diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..6f1f464 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,16 @@ +Please fill out the sections below to help us address your issue. + +### Version of AWS MSK IAM SASL Signer for Go? + +### Version of Go (`go version`)? + +### Kafka Client Library and Version ? + +### MSK Cluster Type - Provisioned or Serverless ? + +### What issue did you see? + +### Steps to reproduce + +If you have an runnable example, please include it. + diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..58d10f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,85 @@ +--- +name: "🐛 Bug Report" +description: Report a bug +title: "(short issue description)" +labels: [bug, needs-triage] +assignees: [] +body: + - type: textarea + id: description + attributes: + label: Describe the bug + description: What is the problem? A clear and concise description of the bug. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Behavior + description: | + What did you expect to happen? + validations: + required: true + - type: textarea + id: current + attributes: + label: Current Behavior + description: | + What actually happened? + + Please include full errors, uncaught exceptions, stack traces, and relevant logs. + If service responses are relevant, please include wire logs. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Reproduction Steps + description: | + Provide a self-contained, concise snippet of code that can be used to reproduce the issue. + For more complex issues provide a repo with the smallest sample that reproduces the bug. + + Avoid including business logic or unrelated code, it makes diagnosis more difficult. + The code sample should be an SSCCE. See http://sscce.org/ for details. In short, please provide a code sample that we can copy/paste, run and reproduce. + validations: + required: true + - type: textarea + id: solution + attributes: + label: Possible Solution + description: | + Suggest a fix/reason for the bug + validations: + required: false + - type: textarea + id: context + attributes: + label: Additional Information/Context + description: | + Anything else that might be relevant for troubleshooting this bug. Providing context helps us come up with a solution that is most useful in the real world. + validations: + required: false + + - type: textarea + id: aws-msk-iam-sasl-signer-go-version + attributes: + label: aws-msk-iam-sasl-signer-go Module Versions Used + description: | + Output of `go mod graph` or `go.mod` file listing the `github.com/aws/*` entries. + validations: + required: true + + - type: input + id: go-version + attributes: + label: Compiler and Version used + description: output of the `go version` command + validations: + required: true + + - type: input + id: operating-system + attributes: + label: Operating System and version + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0f8967a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: 💬 General Question + url: https://github.com/aws/aws-msk-iam-sasl-signer-go/discussions/categories/q-a + about: Please ask and answer questions as a discussion thread \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 0000000..3050baa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,32 @@ +--- +name: "📕 Documentation Issue" +description: Report an issue in the API Reference documentation or Developer Guide +title: "(short issue description)" +labels: [documentation, needs-triage] +assignees: [] +body: + - type: textarea + id: description + attributes: + label: Describe the issue + description: A clear and concise description of the issue. + validations: + required: true + + - type: textarea + id: links + attributes: + label: Links + description: | + Include links to affected documentation page(s). + validations: + required: true + + - type: textarea + id: aws-msk-iam-sasl-signer-go-version + attributes: + label: aws-msk-iam-sasl-signer-go Module Versions Used + description: | + Output of `go mod graph` or `go.mod` file listing the `github.com/aws/*` entries. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..ce6def7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,64 @@ +--- +name: 🚀 Feature Request +description: Suggest an idea for this project +title: "(short issue description)" +labels: [feature-request, needs-triage] +assignees: [] +body: + - type: textarea + id: description + attributes: + label: Describe the feature + description: A clear and concise description of the feature you are proposing. + validations: + required: true + - type: textarea + id: use-case + attributes: + label: Use Case + description: | + Why do you need this feature? For example: "I'm always frustrated when..." + validations: + required: true + - type: textarea + id: solution + attributes: + label: Proposed Solution + description: | + Suggest how to implement the addition or change. Please include prototype/workaround/sketch/reference implementation. + validations: + required: false + - type: textarea + id: other + attributes: + label: Other Information + description: | + Any alternative solutions or features you considered, a more detailed explanation, stack traces, related issues, links for context, etc. + validations: + required: false + - type: checkboxes + id: ack + attributes: + label: Acknowledgements + options: + - label: I may be able to implement this feature request + required: false + - label: This feature might incur a breaking change + required: false + + - type: textarea + id: aws-msk-iam-sasl-signer-go-version + attributes: + label: aws-msk-iam-sasl-signer-go Module Versions Used + description: | + Output of `go mod graph` or `go.mod` file listing the `github.com/aws/*` entries. + validations: + required: true + + - type: input + id: go-version + attributes: + label: Go version used + description: Output of `go version` + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..5cd7ce3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,3 @@ +If the PR addresses an existing bug or feature, please reference it here. + +To help speed up the process and reduce the time to merge please ensure that `Allow edits by maintainers` is checked before submitting your PR. This will allow the project maintainers to make minor adjustments or improvements to the submitted PR, allow us to reduce the roundtrip time for merging your request. diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..9d3b4f2 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,5 @@ +## Reporting a Vulnerability + +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security +via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. +Please do **not** create a public GitHub issue. diff --git a/.github/workflows/closed-issue-message.yml b/.github/workflows/closed-issue-message.yml new file mode 100644 index 0000000..09873e5 --- /dev/null +++ b/.github/workflows/closed-issue-message.yml @@ -0,0 +1,17 @@ +name: Closed Issue Message +on: + issues: + types: [closed] +jobs: + auto_comment: + runs-on: ubuntu-latest + steps: + - uses: aws-actions/closed-issue-message@v1 + with: + # These inputs are both required + repo-token: "${{ secrets.GITHUB_TOKEN }}" + message: | + ### ⚠️COMMENT VISIBILITY WARNING⚠️ + Comments on closed issues are hard for our team to see. + If you need more assistance, please either tag a team member or open a new issue that references this one. + If you wish to keep having a conversation with other community members under this issue feel free to do so. diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..2dafe67 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,55 @@ +name: Go Tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + EACHMODULE_CONCURRENCY: 2 + +jobs: + tests: + name: Latest Go versions tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + go-version: [1.19, "1.20"] + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... + + deprecated-versions-tests: + needs: tests + name: Deprecated Go version Tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + go-version: [ 1.17, 1.18 ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... \ No newline at end of file diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..0134f47 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,49 @@ +name: golangci-lint +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + pull-requests: read + +jobs: + golangci: + strategy: + matrix: + go: [ '1.20' ] + os: [ macos-latest, windows-latest ] + name: lint + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go }} + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Require: The version of golangci-lint to use. + # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. + # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. + version: v1.53 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # + # Note: by default the `.golangci.yml` file should be at the root of the repository. + # The location of the configuration file can be changed by using `--config=` + # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0 + args: --timeout=10m --verbose + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional:The mode to install golangci-lint. It can be 'binary' or 'goinstall'. + # install-mode: "goinstall" \ No newline at end of file diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 0000000..8f6fcc6 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,48 @@ +name: License Scan + +on: [pull_request] + +jobs: + licensescan: + name: License Scan + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9] + + steps: + - name: Checkout target + uses: actions/checkout@v2 + with: + path: signermain + ref: ${{ github.base_ref }} + - name: Checkout this ref + uses: actions/checkout@v2 + with: + path: new-ref + fetch-depth: 0 + - name: Get Diff + run: git --git-dir ./new-ref/.git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > refDiffFiles.txt + - name: Get Target Files + run: git --git-dir ./signermain/.git ls-files | grep -xf refDiffFiles.txt - > targetFiles.txt + - name: Checkout scancode + uses: actions/checkout@v2 + with: + repository: nexB/scancode-toolkit + path: scancode-toolkit + fetch-depth: 1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + # ScanCode + - name: Self-configure scancode + working-directory: ./scancode-toolkit + run: ./scancode --help + - name: Run Scan code on pr ref + run: cat targetFiles.txt | while read filename; do echo ./signermain/$filename; done | xargs ./scancode-toolkit/scancode -l -n 30 --json-pp - | grep short_name | sort | uniq >> old-licenses.txt + - name: Run Scan code on target + run: cat refDiffFiles.txt | while read filename; do echo ./new-ref/$filename; done | xargs ./scancode-toolkit/scancode -l -n 30 --json-pp - | grep short_name | sort | uniq >> new-licenses.txt + # compare + - name: License test + run: if ! cmp old-licenses.txt new-licenses.txt; then echo "Licenses differ! Failing."; exit -1; else echo "Licenses are the same. Success."; exit 0; fi diff --git a/.github/workflows/securityscan.yml b/.github/workflows/securityscan.yml new file mode 100644 index 0000000..00db458 --- /dev/null +++ b/.github/workflows/securityscan.yml @@ -0,0 +1,38 @@ +name: Security Scan + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '00 11 * * 2' + +jobs: + securityscan: + name: Security Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: TruffleHog Secrets Scanner + uses: trufflesecurity/trufflehog@v3.47.0 + with: + path: ./ + base: ${{ github.event.repository.default_branch }} + head: HEAD + extra_args: --debug --only-verified + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: ./... + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: go + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/stale_issue.yml b/.github/workflows/stale_issue.yml new file mode 100644 index 0000000..fdc6f80 --- /dev/null +++ b/.github/workflows/stale_issue.yml @@ -0,0 +1,45 @@ +name: "Close stale issues" + +# Controls when the action will run. +on: + schedule: + - cron: "0 0 * * *" + +jobs: + cleanup: + runs-on: ubuntu-latest + name: Stale issue job + steps: + - uses: aws-actions/stale-issue-cleanup@v3 + with: + # Setting messages to an empty string will cause the automation to skip + # that category + ancient-issue-message: We have noticed this issue has not received attention in 1 year. We will close this issue for now. If you think this is in error, please feel free to comment and reopen the issue. + stale-issue-message: This issue has not received a response in 1 month. If you want to keep this issue open, please just leave a comment below and auto-close will be canceled. + stale-pr-message: Greetings! It looks like this PR hasn’t been active in longer than a month, add a comment or an upvote to prevent automatic closure, or if the issue is already closed, please feel free to open a new one. + + # These labels are required + stale-issue-label: closing-soon + exempt-issue-label: no-autoclose + stale-pr-label: no-pr-activity + exempt-pr-label: awaiting-approval + response-requested-label: response-requested + + # Don't set closed-for-staleness label to skip closing very old issues + # regardless of label + closed-for-staleness-label: closed-for-staleness + + # Issue timing + days-before-stale: 30 + days-before-close: 60 + days-before-ancient: 365 + + # If you don't want to mark a issue as being ancient based on a + # threshold of "upvotes", you can set this here. An "upvote" is + # the total number of +1, heart, hooray, and rocket reactions + # on an issue. + minimum-upvotes-to-exempt: 1 + + repo-token: ${{ secrets.GITHUB_TOKEN }} + # loglevel: DEBUG + # Set dry-run to true to not perform label or close actions. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea95a0c --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +dist +/doc +/doc-staging +.yardoc +Gemfile.lock +/internal +/vendor +/private +.gradle/ +build/ +.idea/ +bin/ +.vscode/ +*.iml diff --git a/.golangci.toml b/.golangci.toml new file mode 100644 index 0000000..9848321 --- /dev/null +++ b/.golangci.toml @@ -0,0 +1,26 @@ +[run] +concurrency = 4 +timeout = "1m" +issues-exit-code = 0 +modules-download-mode = "readonly" +allow-parallel-runners = true +skip-dirs = ["internal"] +skip-dirs-use-default = true +[output] +format = "github-actions" + +[linters-settings.cyclop] +skip-tests = false + +[linters-settings.errcheck] +check-blank = true + +[linters] +disable-all = true +enable = ["errcheck"] +fast = false + +[issues] +exclude-use-default = false + +# Refer config definitions at https://golangci-lint.run/usage/configuration/#config-file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8b4a63a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [1.0.0] - 2023-11-09 + +### Added + +- Release first version of library diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..dfe12ad --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,3 @@ +# Add core contributors to all PRs by default + +* @aws/amazon-managed-streaming-for-apache-kafka diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4b6a1c..f47ccc9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,59 +1,137 @@ -# Contributing Guidelines +# Contributing to the AWS MSK IAM SASL Signer for Go -Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional -documentation, we greatly value feedback and contributions from our community. +Thank you for your interest in contributing to the AWS MSK IAM SASL Signer for Go! +We work hard to provide a high-quality and useful Signer module that can work with any Kafka Client library written in Go, +and we greatly value feedback and contributions from our community. Whether it's a bug report, +new feature, correction, or additional documentation, we welcome your issues +and pull requests. Please read through this document before submitting any +[issues] or [pull requests][pr] to ensure we have all the necessary information to +effectively respond to your bug report or contribution. -Please read through this document before submitting any issues or pull requests to ensure we have all the necessary -information to effectively respond to your bug report or contribution. +Jump To: +* [Bug Reports](#bug-reports) +* [Feature Requests](#feature-requests) +* [Code Contributions](#code-contributions) -## Reporting Bugs/Feature Requests +## How to contribute -We welcome you to use the GitHub issue tracker to report bugs or suggest features. +*Before you send us a pull request, please be sure that:* -When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already -reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: +1. You're working from the latest source on the `main` branch. +2. You check existing open, and recently closed, pull requests to be sure + that someone else hasn't already addressed the problem. +3. You create an issue before working on a contribution that will take a + significant amount of your time. -* A reproducible test case or series of steps -* The version of our code being used -* Any modifications you've made relevant to the bug -* Anything unusual about your environment or deployment - - -## Contributing via Pull Requests -Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: - -1. You are working against the latest source on the *main* branch. -2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. -3. You open an issue to discuss any significant work - we would hate for your time to be wasted. - -To send us a pull request, please: +*Creating a Pull Request* 1. Fork the repository. -2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. -3. Ensure local tests pass. -4. Commit to your fork using clear commit messages. -5. Send us a pull request, answering any default questions in the pull request interface. -6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. - -GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and -[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). - - -## Finding contributions to work on -Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. - - -## Code of Conduct -This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). -For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact -opensource-codeofconduct@amazon.com with any additional questions or comments. - - -## Security issue notifications -If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. - +2. In your fork, make your change in a branch that's based on this repo's `main` branch. +3. Commit the change to your fork, using a clear and descriptive commit message. +4. Create a pull request, answering any questions in the pull request form. + +For contributions that will take a significant amount of time, open a new +issue to pitch your idea before you get started. Explain the problem and +describe the content you want to see added to the documentation. Let us know +if you'll write it yourself or if you'd like us to help. We'll discuss your +proposal with you and let you know whether we're likely to accept it. + +## Bug Reports + +You can file bug reports against the Signer module on the [GitHub issues][issues] page. + +If you are filing a report for a bug or regression in the module, it's extremely +helpful to provide as much information as possible when opening the original +issue. This helps us reproduce and investigate the possible bug without having +to wait for this extra information to be provided. Please read the following +guidelines prior to filing a bug report. + +1. Search through existing [issues][] to ensure that your specific issue has + not yet been reported. If it is a common issue, it is likely there is + already a bug report for your problem. + +2. Ensure that you have tested the latest version of the Signer module. Although you + may have an issue against an older version of the Signer module, we cannot provide + bug fixes for old versions. It's also possible that the bug may have been + fixed in the latest release. + +3. Provide as much information about your environment, Signer module version, Kafka library name and version and + relevant dependencies as possible. For example, let us know what version + of Go you are using, which and version of the operating system, and + the environment your code is running in. e.g Container. + +4. Provide a minimal test case that reproduces your issue or any error + information you related to your problem. We can provide feedback much + more quickly if we know what operations you are calling in the Signer module. If + you cannot provide a full test case, provide as much code as you can + to help us diagnose the problem. Any relevant information should be provided + as well, like whether this is a persistent issue, or if it only occurs + some of the time. + +## Feature Requests + +Open an [issue][issues] with the following: + +* A short, descriptive title. Ideally, other community members should be able + to get a good idea of the feature just from reading the title. +* A detailed description of the proposed feature. + * Why it should be added to the module. + * If possible, example code to illustrate how it should work. +* Use Markdown to make the request easier to read; +* If you intend to implement this feature, indicate that you'd like to the issue to be assigned to you. + +## Code Contributions + +We are always happy to receive code and documentation contributions to the Signer module. +Please be aware of the following notes prior to opening a pull request: + +1. The Signer module is released under the [Apache license][license]. Any code you submit + will be released under that license. For substantial contributions, we may + ask you to sign a [Contributor License Agreement (CLA)][cla]. + +2. If you would like to implement support for a significant feature that is not + yet available in the Signer module, please talk to us beforehand to avoid any + duplication of effort. + +3. Wherever possible, pull requests should contain tests as appropriate. + Bugfixes should contain tests that exercise the corrected behavior (i.e., the + test should fail without the bugfix and pass with it), and new features + should be accompanied by tests exercising the feature. + +4. Pull requests that contain failing tests will not be merged until the test + failures are addressed. Pull requests that cause a significant drop in the + Signer module's test coverage percentage are unlikely to be merged until tests have + been added. + +### Testing + +To run the tests locally, running the `make unit` command will `go get` the +Signer module's testing dependencies, and run vet, link and unit tests for the Signer module. + +``` +make unit +``` + +### Changelog Documents + +You can see all release changes in the `CHANGELOG.md` file at the root of the +repository. The release notes added to this file will contain service client +updates, and major Signer module changes. When submitting a pull request please include an entry in `CHANGELOG_PENDING.md` under the appropriate changelog type so your changelog entry is included on the following release. + +#### Changelog Types + +* `Signer module Features` - For major additive features, internal changes that have +outward impact, or updates to the Signer module foundations. This will result in a minor +version change. +* `Signer module Enhancements` - For minor additive features or incremental sized changes. +This will result in a patch version change. +* `Signer module Bugs` - For minor changes that resolve an issue. This will result in a +patch version change. -## Licensing +[issues]: https://github.com/aws/aws-msk-iam-sasl-signer-go/issues +[pr]: https://github.com/aws/aws-msk-iam-sasl-signer-go/pulls +[license]: http://aws.amazon.com/apache2.0/ +[cla]: http://en.wikipedia.org/wiki/Contributor_License_Agreement +[releasenotes]: https://github.com/aws/aws-msk-iam-sasl-signer-go/releases -See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..5efe55e --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,10 @@ +Open Discussions +--- +The following issues are currently open for community feedback. +All discourse must adhere to the [Code of Conduct] policy. + +Past Discussions +--- +The issues listed here are for documentation purposes, and is used to capture issues and their associated discussions. + +[Code of Conduct]: https://github.com/aws/aws-msk-iam-sasl-signer-go/blob/main/CODE_OF_CONDUCT.md diff --git a/README.md b/README.md index 847260c..23b552a 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,329 @@ -## My Project +# AWS MSK IAM SASL Signer for Go -TODO: Fill this README out! +[![Go Build status](https://github.com/aws/aws-msk-iam-sasl-signer-go/actions/workflows/go.yml/badge.svg?branch=main)](https://github.com/aws/aws-msk-iam-sasl-signer-go/actions/workflows/go.yml) [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/aws/aws-msk-iam-sasl-signer-go/blob/main/LICENSE.txt) +[![Security Scan](https://github.com/aws/aws-msk-iam-sasl-signer-go/actions/workflows/securityscan.yml/badge.svg?branch=main)](https://github.com/aws/aws-msk-iam-sasl-signer-go/actions/workflows/securityscan.yml) -Be sure to: +`aws-msk-iam-sasl-signer-go` is the AWS MSK IAM SASL Signer for Go programming language. -* Change the title in this README -* Edit your repository description on GitHub +The AWS MSK IAM SASL Signer for Go requires a minimum version of `Go 1.17`. -## Security +Check out the [release notes](https://github.com/aws/aws-msk-iam-sasl-signer-go/blob/main/CHANGELOG.md) for information about the latest bug +fixes, updates, and features added to the library. -See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. +Jump To: +* [Getting Started](#getting-started) +* [Getting Help](#getting-help) +* [Contributing](#feedback-and-contributing) +* [More Resources](#resources) -## License -This project is licensed under the Apache-2.0 License. +## Getting started +To get started working with the AWS MSK IAM SASL Signer for Go with your Kafka client library please follow below code sample - +###### Add Dependencies +```sh +$ go get github.com/aws/aws-msk-iam-sasl-signer-go +``` + +###### Write Code + +For example, you can use the signer library to generate IAM default credentials based OAUTH token with [IBM sarama library](https://github.com/IBM/sarama) as below - + +```go +package main + +import ( + "context" + "crypto/tls" + "log" + "os" + "os/signal" + "time" + + "github.com/aws/aws-msk-iam-sasl-signer-go/signer" + "github.com/Shopify/sarama" +) + +var ( + kafkaBrokers = []string{""} + KafkaTopic = "" + enqueued int +) + +type MSKAccessTokenProvider struct { +} + +func (m *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) { + token, _, err := signer.GenerateAuthToken(context.TODO(), "") + return &sarama.AccessToken{Token: token}, err} + +func main() { + sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags) + producer, err := setupProducer() + if err != nil { + panic(err) + } else { + log.Println("Kafka AsyncProducer up and running!") + } + + // Trap SIGINT to trigger a graceful shutdown. + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt) + + produceMessages(producer, signals) + + log.Printf("Kafka AsyncProducer finished with %d messages produced.", enqueued) +} + +// setupProducer will create a AsyncProducer and returns it +func setupProducer() (sarama.AsyncProducer, error){ + // Set the SASL/OAUTHBEARER configuration + config := sarama.NewConfig() + config.Net.SASL.Enable = true + config.Net.SASL.Mechanism = sarama.SASLTypeOAuth + config.Net.SASL.TokenProvider = &MSKAccessTokenProvider{} + + tlsConfig := tls.Config{} + config.Net.TLS.Enable = true + config.Net.TLS.Config = &tlsConfig + return sarama.NewAsyncProducer(kafkaBrokers, config) +} + +// produceMessages will send 'testing 123' to KafkaTopic each second, until receive a os signal to stop e.g. control + c +// by the user in terminal +func produceMessages(producer sarama.AsyncProducer, signals chan os.Signal) { + for { + time.Sleep(time.Second) + message := &sarama.ProducerMessage{Topic: KafkaTopic, Value: sarama.StringEncoder("testing 123")} + select { + case producer.Input() <- message: + enqueued++ + log.Println("New Message produced") + case <-signals: + producer.AsyncClose() // Trigger a shutdown of the producer. + return + } + } +} +``` + +Consumer - + +```go +package main + +import ( + "context" + "crypto/tls" + "fmt" + "log" + "os" + "os/signal" + + "github.com/awslabs/aws-msk-iam-sasl-signer-go/signer" + "github.com/Shopify/sarama" +) + +var ( + kafkaBrokers = []string{""} + KafkaTopic = "" +) + +type MSKAccessTokenProvider struct { +} + +func (m *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) { + token, _, err := signer.GenerateAuthToken(context.TODO(), "") + return &sarama.AccessToken{Token: token}, err +} + +func main() { + sarama.Logger = log.New(os.Stdout, "[sarama] ", log.LstdFlags) + consumer, err := setUpConsumer() + if err != nil { + panic(err) + } else { + log.Println("Kafka Consumer is up and running!") + } + + defer func() { + if err := consumer.Close(); err != nil { + log.Printf("Error closing consumer: %w", err) + } + }() + + consumeMessages(consumer) +} + +func setUpConsumer() (sarama.Consumer, error) { + // Set the SASL/OAUTHBEARER configuration + config := sarama.NewConfig() + config.Net.SASL.Enable = true + config.Net.SASL.Mechanism = sarama.SASLTypeOAuth + config.Net.SASL.TokenProvider = &MSKAccessTokenProvider{} + + tlsConfig := tls.Config{} + config.Net.TLS.Enable = true + config.Net.TLS.Config = &tlsConfig + return sarama.NewConsumer(kafkaBrokers, config) +} + +func consumeMessages(consumer sarama.Consumer) { + partitions, err := consumer.Partitions(KafkaTopic) + if err != nil { + log.Fatalf("Failed to retrieve partitions for topic %s: %v", KafkaTopic, err) + } + + consumers := make(chan *sarama.ConsumerMessage) + errors := make(chan *sarama.ConsumerError) + + // Create a partition consumer and goroutine for each partition + for _, partition := range partitions { + partitionConsumer, err := consumer.ConsumePartition(KafkaTopic, partition, sarama.OffsetNewest) + if err != nil { + log.Fatalf("Failed to create partition consumer for topic %s, partition %d: %v", KafkaTopic, partition, err) + } + + go func(KafkaTopic string, partitionConsumer sarama.PartitionConsumer) { + for { + select { + case consumerError := <-partitionConsumer.Errors(): + errors <- consumerError + + case msg := <-partitionConsumer.Messages(): + consumers <- msg + } + } + }(KafkaTopic, partitionConsumer) + } + + signals := make(chan os.Signal, 1) + signal.Notify(signals, os.Interrupt) + + msgCount := 0 + + doneCh := make(chan struct{}) + go func() { + for { + select { + case msg := <-consumers: + msgCount++ + fmt.Println("Received message : ", string(msg.Key), string(msg.Value)) + case consumerError := <-errors: + msgCount++ + fmt.Println("Received consumerError ", string(consumerError.Topic), string(consumerError.Partition), consumerError.Err) + doneCh <- struct{}{} + case <-signals: + fmt.Println("Interrupt is detected") + doneCh <- struct{}{} + } + } + }() + + <-doneCh + fmt.Println("Processed", msgCount, "messages") +} + + +``` + +* To use IAM credentials from a named profile, update the Token() function: +```go +func (t *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) { + token, _, err := signer.GenerateAuthTokenFromProfile(context.TODO(), "", "") + return &sarama.AccessToken{Token: token}, err +} +``` + +* To use IAM credentials by assuming a IAM Role using sts, update the Token() function: + +```go +func (t *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) { + token, _, err := signer.GenerateAuthTokenFromRole(context.TODO(), "", "", "my-sts-session-name") + return &sarama.AccessToken{Token: token}, err +} +``` +* To use IAM credentials from a credentials provider, update the Token() function: +```go +func (t *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) { + token, _, err := signer.GenerateAuthTokenFromCredentialsProvider(context.TODO(), "", ) + return &sarama.AccessToken{Token: token}, err +} +``` + + +###### Compile and Execute +```sh +$ go build +$ go run . +``` + +###### Test +```sh +$ cd signer +$ go test +``` + +## Troubleshooting +### Finding out which identity is being used +You may receive an `Access denied` error and there may be some doubt as to which credential is being exactly used. The credential may be sourced from a role ARN, EC2 instance profile, credential profile etc. +You can set the field `AwsDebugCreds` set to true before getting the token: + +```go + signer.AwsDebugCreds = true +``` +the client library will print a debug log of the form: +``` +Credentials Identity: {UserId: ABCD:test124, Account: 1234567890, Arn: arn:aws:sts::1234567890:assumed-role/abc/test124} +``` + +The log line provides the IAM Account, IAM user id and the ARN of the IAM Principal corresponding to the credential being used. + +Please note that the log level should also be set to DEBUG for this information to be logged. It is not recommended to run with AwsDebugCreds=true since it makes an additional remote call. + +## Getting Help + +Please use these community resources for getting help. We use the GitHub issues +for tracking bugs and feature requests. + +* Ask us a [question](https://github.com/aws/aws-msk-iam-sasl-signer-go/discussions/new?category=q-a) or open a [discussion](https://github.com/aws/aws-msk-iam-sasl-signer-go/discussions/new?category=general). +* If you think you may have found a bug, please open an [issue](https://github.com/aws/aws-msk-iam-sasl-signer-go/issues/new/choose). +* Open a support ticket with [AWS Support](http://docs.aws.amazon.com/awssupport/latest/user/getting-started.html). + +This repository provides a pluggable library with any Go Kafka client for SASL/OAUTHBEARER mechanism. For more information about SASL/OAUTHBEARER mechanism please go to [KIP 255](https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=75968876). + +### Opening Issues + +If you encounter a bug with the AWS MSK IAM SASL Signer for Go we would like to hear about it. +Search the [existing issues][Issues] and see +if others are also experiencing the same issue before opening a new issue. Please +include the version of AWS MSK IAM SASL Signer for Go, Go language, and OS you’re using. Please +also include reproduction case when appropriate. + +The GitHub issues are intended for bug reports and feature requests. For help +and questions with using AWS MSK IAM SASL Signer for Go, please make use of the resources listed +in the [Getting Help](#getting-help) section. +Keeping the list of open issues lean will help us respond in a timely manner. + +## Feedback and contributing + +The AWS MSK IAM SASL Signer for Go will use GitHub [Issues] to track feature requests and issues with the library. In addition, we'll use GitHub [Projects] to track large tasks spanning multiple pull requests, such as refactoring the library's internal request lifecycle. You can provide feedback to us in several ways. + +**GitHub issues**. To provide feedback or report bugs, file GitHub [Issues] on the library. This is the preferred mechanism to give feedback so that other users can engage in the conversation, +1 issues, etc. Issues you open will be evaluated, and included in our roadmap for the GA launch. + +**Contributing**. You can open pull requests for fixes or additions to the AWS MSK IAM SASL Signer for Go. All pull requests must be submitted under the Apache 2.0 license and will be reviewed by a team member before being merged in. Accompanying unit tests, where possible, are appreciated. + +## Resources + +[Service Documentation](https://docs.aws.amazon.com/msk/latest/developerguide/getting-started.html) - Use this +documentation to learn how to interface with AWS MSK. + +[Issues] - Report issues, submit pull requests, and get involved + (see [Apache 2.0 License][license]) + +[Dep]: https://github.com/golang/dep +[Issues]: https://github.com/aws/aws-msk-iam-sasl-signer-go/issues +[Projects]: https://github.com/aws/aws-msk-iam-sasl-signer-go/projects +[CHANGELOG]: https://github.com/aws/aws-msk-iam-sasl-signer-go/blob/main/CHANGELOG.md +[design]: https://github.com/aws/aws-msk-iam-sasl-signer-go/blob/main/DESIGN.md +[license]: http://aws.amazon.com/apache2.0/ diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 0000000..8004fab --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,13 @@ +version: 0.2 + +phases: + build: + commands: + - echo Build started on `date` + - export GOPATH=/go + - export CODEBUILD_ROOT=`pwd` + - cd signer + - go test + post_build: + commands: + - echo Build completed on `date` diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..f158c30 --- /dev/null +++ b/doc.go @@ -0,0 +1 @@ +package signer diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e58e6e2 --- /dev/null +++ b/go.mod @@ -0,0 +1,25 @@ +module aws-msk-iam-sasl-signer-go + +go 1.17 + +require ( + github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 + github.com/stretchr/testify v1.8.4 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect + github.com/aws/smithy-go v1.13.5 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..27afa25 --- /dev/null +++ b/go.sum @@ -0,0 +1,46 @@ +github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k= +github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw= +github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/signer/msk_auth_token_provider.go b/signer/msk_auth_token_provider.go new file mode 100644 index 0000000..1c2670d --- /dev/null +++ b/signer/msk_auth_token_provider.go @@ -0,0 +1,305 @@ +package signer + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + + "log" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +const ( + ActionType = "Action" // ActionType represents the key for the action type in the request. + ActionName = "kafka-cluster:Connect" // ActionName represents the specific action name for connecting to a Kafka cluster. + SigningName = "kafka-cluster" // SigningName represents the signing name for the Kafka cluster. + UserAgentKey = "User-Agent" // UserAgentKey represents the key for the User-Agent parameter in the request. + LibName = "aws-msk-iam-sasl-signer-go" // LibName represents the name of the library. + ExpiresQueryKey = "X-Amz-Expires" // ExpiresQueryKey represents the key for the expiration time in the query parameters. + DefaultSessionName = "MSKSASLDefaultSession" // DefaultSessionName represents the default session name for assuming a role. + DefaultExpirySeconds = 900 // DefaultExpirySeconds represents the default expiration time in seconds. +) + +var ( + endpointURLTemplate = "kafka.%s.amazonaws.com" // endpointURLTemplate represents the template for the Kafka endpoint URL + AwsDebugCreds = false // AwsDebugCreds flag indicates whether credentials should be debugged +) + +// GenerateAuthToken generates base64 encoded signed url as auth token from default credentials. +// Loads the IAM credentials from default credentials provider chain. +func GenerateAuthToken(ctx context.Context, region string) (string, int64, error) { + credentials, err := loadDefaultCredentials(ctx, region) + + if err != nil { + return "", 0, fmt.Errorf("failed to load credentials: %w", err) + } + + return constructAuthToken(ctx, region, credentials) +} + +// GenerateAuthTokenFromProfile generates base64 encoded signed url as auth token by loading IAM credentials from an AWS named profile. +func GenerateAuthTokenFromProfile(ctx context.Context, region string, awsProfile string) (string, int64, error) { + credentials, err := loadCredentialsFromProfile(ctx, region, awsProfile) + + if err != nil { + return "", 0, fmt.Errorf("failed to load credentials: %w", err) + } + + return constructAuthToken(ctx, region, credentials) +} + +// GenerateAuthTokenFromRole generates base64 encoded signed url as auth token by loading IAM credentials from an aws role Arn +func GenerateAuthTokenFromRole( + ctx context.Context, region string, roleArn string, stsSessionName string, +) (string, int64, error) { + if stsSessionName == "" { + stsSessionName = DefaultSessionName + } + credentials, err := loadCredentialsFromRoleArn(ctx, region, roleArn, stsSessionName) + + if err != nil { + return "", 0, fmt.Errorf("failed to load credentials: %w", err) + } + + return constructAuthToken(ctx, region, credentials) +} + +// GenerateAuthTokenFromCredentialsProvider generates base64 encoded signed url as auth token by loading IAM credentials +// from an aws credentials provider +func GenerateAuthTokenFromCredentialsProvider( + ctx context.Context, region string, credentialsProvider aws.CredentialsProvider, +) (string, int64, error) { + credentials, err := loadCredentialsFromCredentialsProvider(ctx, credentialsProvider) + + if err != nil { + return "", 0, fmt.Errorf("failed to load credentials: %w", err) + } + + return constructAuthToken(ctx, region, credentials) +} + +// Loads credentials from the default credential chain. +func loadDefaultCredentials(ctx context.Context, region string) (*aws.Credentials, error) { + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + + if err != nil { + return nil, fmt.Errorf("unable to load SDK config: %w", err) + } + + return loadCredentialsFromCredentialsProvider(ctx, cfg.Credentials) +} + +// Loads credentials from a named aws profile. +func loadCredentialsFromProfile(ctx context.Context, region string, awsProfile string) (*aws.Credentials, error) { + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(region), + config.WithSharedConfigProfile(awsProfile), + ) + + if err != nil { + return nil, fmt.Errorf("unable to load SDK config: %w", err) + } + + return loadCredentialsFromCredentialsProvider(ctx, cfg.Credentials) +} + +// Loads credentials from a named by assuming the passed role. +// This implementation creates a new sts client for every call to get or refresh token. In order to avoid this, please +// use your own credentials provider. +// If you wish to use regional endpoint, please pass your own credentials provider. +func loadCredentialsFromRoleArn( + ctx context.Context, region string, roleArn string, stsSessionName string, +) (*aws.Credentials, error) { + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + + if err != nil { + return nil, fmt.Errorf("unable to load SDK config: %w", err) + } + + stsClient := sts.NewFromConfig(cfg) + + assumeRoleInput := &sts.AssumeRoleInput{ + RoleArn: aws.String(roleArn), + RoleSessionName: aws.String(stsSessionName), + } + assumeRoleOutput, err := stsClient.AssumeRole(ctx, assumeRoleInput) + if err != nil { + return nil, fmt.Errorf("unable to assume role, %s: %w", roleArn, err) + } + + //Create new aws.Credentials instance using the credentials from AssumeRoleOutput.Credentials + creds := aws.Credentials{ + AccessKeyID: *assumeRoleOutput.Credentials.AccessKeyId, + SecretAccessKey: *assumeRoleOutput.Credentials.SecretAccessKey, + SessionToken: *assumeRoleOutput.Credentials.SessionToken, + } + + return &creds, nil +} + +// Loads credentials from the credentials provider +func loadCredentialsFromCredentialsProvider( + ctx context.Context, credentialsProvider aws.CredentialsProvider, +) (*aws.Credentials, error) { + creds, err := credentialsProvider.Retrieve(ctx) + return &creds, err +} + +// Constructs Auth Token. +func constructAuthToken(ctx context.Context, region string, credentials *aws.Credentials) (string, int64, error) { + endpointURL := fmt.Sprintf(endpointURLTemplate, region) + + if credentials == nil || credentials.AccessKeyID == "" || credentials.SecretAccessKey == "" { + return "", 0, fmt.Errorf("aws credentials cannot be empty") + } + + if AwsDebugCreds { + logCallerIdentity(ctx, region, *credentials) + } + + req, err := buildRequest(DefaultExpirySeconds, endpointURL) + if err != nil { + return "", 0, fmt.Errorf("failed to build request for signing: %w", err) + } + + signedURL, err := signRequest(ctx, req, region, credentials) + if err != nil { + return "", 0, fmt.Errorf("failed to sign request with aws sig v4: %w", err) + } + + expirationTimeMs, err := getExpirationTimeMs(signedURL) + if err != nil { + return "", 0, fmt.Errorf("failed to extract expiration from signed url: %w", err) + } + + signedURLWithUserAgent, err := addUserAgent(signedURL) + if err != nil { + return "", 0, fmt.Errorf("failed to add user agent to the signed url: %w", err) + } + + return base64Encode(signedURLWithUserAgent), expirationTimeMs, nil +} + +// Build https request with query parameters in order to sign. +func buildRequest(expirySeconds int, endpointURL string) (*http.Request, error) { + query := url.Values{ + ActionType: {ActionName}, + ExpiresQueryKey: {strconv.FormatInt(int64(expirySeconds), 10)}, + } + + authURL := url.URL{ + Host: endpointURL, + Scheme: "https", + Path: "/", + RawQuery: query.Encode(), + } + + return http.NewRequest(http.MethodGet, authURL.String(), nil) +} + +// Sign request with aws sig v4. +func signRequest(ctx context.Context, req *http.Request, region string, credentials *aws.Credentials) (string, error) { + signer := v4.NewSigner() + signedURL, _, err := signer.PresignHTTP(ctx, *credentials, req, + calculateSHA256Hash(""), + SigningName, + region, + time.Now().UTC(), + ) + + return signedURL, err +} + +// Parses the URL and gets the expiration time in millis associated with the signed url +func getExpirationTimeMs(signedURL string) (int64, error) { + parsedURL, err := url.Parse(signedURL) + + if err != nil { + return 0, fmt.Errorf("failed to parse the signed url: %w", err) + } + + params := parsedURL.Query() + date, err := time.Parse("20060102T150405Z", params.Get("X-Amz-Date")) + + if err != nil { + return 0, fmt.Errorf("failed to parse the 'X-Amz-Date' param from signed url: %w", err) + } + + signingTimeMs := date.UnixNano() / int64(time.Millisecond) + expiryDurationSeconds, err := strconv.ParseInt(params.Get("X-Amz-Expires"), 10, 64) + + if err != nil { + return 0, fmt.Errorf("failed to parse the 'X-Amz-Expires' param from signed url: %w", err) + } + + expiryDurationMs := expiryDurationSeconds * 1000 + expiryMs := signingTimeMs + expiryDurationMs + return expiryMs, nil +} + +// Calculate sha256Hash and hex encode it. +func calculateSHA256Hash(input string) string { + hash := sha256.Sum256([]byte(input)) + return hex.EncodeToString(hash[:]) +} + +// Base64 encode with raw url encoding. +func base64Encode(signedURL string) string { + signedURLBytes := []byte(signedURL) + return base64.RawURLEncoding.EncodeToString(signedURLBytes) +} + +// Add user agent to the signed url +func addUserAgent(signedURL string) (string, error) { + parsedSignedURL, err := url.Parse(signedURL) + + if err != nil { + return "", fmt.Errorf("failed to parse signed url: %w", err) + } + + query := parsedSignedURL.Query() + userAgent := strings.Join([]string{LibName, version, runtime.Version()}, "/") + query.Set(UserAgentKey, userAgent) + parsedSignedURL.RawQuery = query.Encode() + + return parsedSignedURL.String(), nil +} + +// Log caller identity to debug which credentials are being picked up +func logCallerIdentity(ctx context.Context, region string, awsCredentials aws.Credentials) { + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(region), + config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ + Value: awsCredentials, + }), + ) + if err != nil { + log.Printf("failed to load AWS configuration: %v", err) + } + + stsClient := sts.NewFromConfig(cfg) + + callerIdentity, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + + if err != nil { + log.Printf("failed to get caller identity: %v", err) + } + + log.Printf("Credentials Identity: {UserId: %s, Account: %s, Arn: %s}\n", + *callerIdentity.UserId, + *callerIdentity.Account, + *callerIdentity.Arn) +} diff --git a/signer/msk_auth_token_provider_test.go b/signer/msk_auth_token_provider_test.go new file mode 100644 index 0000000..da552d1 --- /dev/null +++ b/signer/msk_auth_token_provider_test.go @@ -0,0 +1,243 @@ +package signer + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/stretchr/testify/assert" +) + +var ( + TestRegion = "us-west-2" + TestEndpoint = "kafka.us-west-2.amazonaws.com" + Ctx = context.TODO() +) + +// Provides mocked credentials. +type MockCredentialsProvider struct { + credentials aws.Credentials +} + +func (t MockCredentialsProvider) Retrieve(ctx context.Context) (aws.Credentials, error) { + return t.credentials, nil +} + +func TestCalculatePayloadHashForSigning(t *testing.T) { + sha256HashForEmptyString := calculateSHA256Hash("") + assert.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", sha256HashForEmptyString) + + sha256HashForTestString := calculateSHA256Hash("test") + assert.Equal(t, "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", sha256HashForTestString) +} + +func TestAddUserAgent(t *testing.T) { + signedURL := "https://kafka.us-west-2.amazonaws.com/?Action=kafka-cluster%3AConnect" + result, err := addUserAgent(signedURL) + + assert.NoError(t, err) + assert.True(t, strings.HasPrefix(result, fmt.Sprintf("%s&%s=%s", signedURL, UserAgentKey, LibName))) +} + +func TestAddUserAgentWithInvalidURL(t *testing.T) { + signedURL := ":invalidURL:" + result, err := addUserAgent(signedURL) + + assert.Error(t, err) + assert.Equal(t, "", result) +} + +func TestLoadDefaultCredentials(t *testing.T) { + mockCreds := aws.Credentials{ + AccessKeyID: "MOCK-ACCESS-KEY", + SecretAccessKey: "MOCK-SECRET-KEY", + } + + os.Setenv("AWS_ACCESS_KEY_ID", mockCreds.AccessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", mockCreds.SecretAccessKey) + + creds, err := loadDefaultCredentials(Ctx, TestRegion) + assert.NoError(t, err) + assert.NotNil(t, creds) + assert.Equal(t, mockCreds.AccessKeyID, creds.AccessKeyID) + assert.Equal(t, mockCreds.SecretAccessKey, creds.SecretAccessKey) + + // Clean-up env variables. + os.Unsetenv("AWS_ACCESS_KEY_ID") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") +} + +func TestConstructAuthToken(t *testing.T) { + mockCreds := aws.Credentials{ + AccessKeyID: "MOCK-ACCESS-KEY", + SecretAccessKey: "MOCK-SECRET-KEY", + SessionToken: "MOCK-SESSION-TOKEN", + } + + token, expiryMs, err := constructAuthToken(Ctx, TestRegion, &mockCreds) + + assert.NoError(t, err) + assert.NotNil(t, token) + assert.NotEqual(t, int64(0), expiryMs) + + decodedSignedURLBytes, err := base64.RawURLEncoding.DecodeString(token) + assert.NoError(t, err) + + decodedSignedURL := string(decodedSignedURLBytes) + + parsedURL, err := url.Parse(decodedSignedURL) + assert.NoError(t, err) + assert.NotNil(t, parsedURL) + assert.Equal(t, parsedURL.Scheme, "https") + assert.Equal(t, parsedURL.Host, TestEndpoint) + + params := parsedURL.Query() + + assert.Equal(t, params.Get("Action"), "kafka-cluster:Connect") + assert.Equal(t, params.Get("X-Amz-Algorithm"), "AWS4-HMAC-SHA256") + assert.Equal(t, params.Get("X-Amz-Expires"), "900") + assert.Equal(t, params.Get("X-Amz-Security-Token"), mockCreds.SessionToken) + assert.Equal(t, params.Get("X-Amz-SignedHeaders"), "host") + credential := params.Get("X-Amz-Credential") + splitCredential := strings.Split(credential, "/") + assert.Equal(t, splitCredential[0], mockCreds.AccessKeyID) + assert.Equal(t, splitCredential[2], TestRegion) + assert.Equal(t, splitCredential[3], "kafka-cluster") + assert.Equal(t, splitCredential[4], "aws4_request") + date, err := time.Parse("20060102T150405Z", params.Get("X-Amz-Date")) + assert.NoError(t, err) + assert.True(t, date.Before(time.Now().UTC())) + assert.True(t, strings.HasPrefix(params.Get(UserAgentKey), "aws-msk-iam-sasl-signer-go/")) +} + +func TestGenerateAuthTokenEmptyCredentials(t *testing.T) { + mockCreds := aws.AnonymousCredentials{} + + token, expiryMs, err := GenerateAuthTokenFromCredentialsProvider(Ctx, TestRegion, &mockCreds) + + assert.Error(t, err) + assert.Equal(t, token, "") + assert.Equal(t, int64(0), expiryMs) +} + +func TestGenerateAuthToken(t *testing.T) { + mockCreds := aws.Credentials{ + AccessKeyID: "TEST-ACCESS-KEY", + SecretAccessKey: "TEST-SECRET-KEY", + SessionToken: "TEST-SESSION-TOKEN", + } + + os.Setenv("AWS_ACCESS_KEY_ID", mockCreds.AccessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", mockCreds.SecretAccessKey) + os.Setenv("AWS_SESSION_TOKEN", mockCreds.SessionToken) + + token, expiryMs, err := GenerateAuthToken(Ctx, TestRegion) + + assert.NoError(t, err) + assert.NotNil(t, token) + assert.NotEqual(t, int64(0), expiryMs) + + decodedSignedURLBytes, err := base64.RawURLEncoding.DecodeString(token) + assert.NoError(t, err) + + decodedSignedURL := string(decodedSignedURLBytes) + + parsedURL, err := url.Parse(decodedSignedURL) + assert.NoError(t, err) + assert.NotNil(t, parsedURL) + assert.Equal(t, parsedURL.Scheme, "https") + assert.Equal(t, parsedURL.Host, TestEndpoint) + + params := parsedURL.Query() + + assert.Equal(t, params.Get("Action"), "kafka-cluster:Connect") + assert.Equal(t, params.Get("X-Amz-Algorithm"), "AWS4-HMAC-SHA256") + assert.Equal(t, params.Get("X-Amz-Expires"), "900") + assert.Equal(t, params.Get("X-Amz-Security-Token"), mockCreds.SessionToken) + assert.Equal(t, params.Get("X-Amz-SignedHeaders"), "host") + credential := params.Get("X-Amz-Credential") + splitCredential := strings.Split(credential, "/") + assert.Equal(t, splitCredential[0], mockCreds.AccessKeyID) + assert.Equal(t, splitCredential[2], TestRegion) + assert.Equal(t, splitCredential[3], "kafka-cluster") + assert.Equal(t, splitCredential[4], "aws4_request") + date, err := time.Parse("20060102T150405Z", params.Get("X-Amz-Date")) + assert.NoError(t, err) + assert.True(t, date.Before(time.Now().UTC())) + assert.True(t, strings.HasPrefix(params.Get(UserAgentKey), "aws-msk-iam-sasl-signer-go/")) + + // Clean-up env variables. + os.Unsetenv("AWS_ACCESS_KEY_ID") + os.Unsetenv("AWS_SECRET_ACCESS_KEY") + os.Unsetenv("AWS_SESSION_TOKEN") +} + +func TestGenerateAuthTokenWithCredentialsProvider(t *testing.T) { + mockCreds := aws.Credentials{ + AccessKeyID: "TEST-MY-ACCESS-KEY", + SecretAccessKey: "TEST-MY-SECRET-KEY", + } + + mockCredentialsProvider := MockCredentialsProvider{credentials: mockCreds} + + token, expiryMs, err := GenerateAuthTokenFromCredentialsProvider(Ctx, TestRegion, mockCredentialsProvider) + + assert.NoError(t, err) + assert.NotNil(t, token) + assert.NotEqual(t, int64(0), expiryMs) + + decodedSignedURLBytes, err := base64.RawURLEncoding.DecodeString(token) + assert.NoError(t, err) + + decodedSignedURL := string(decodedSignedURLBytes) + + parsedURL, err := url.Parse(decodedSignedURL) + assert.NoError(t, err) + assert.NotNil(t, parsedURL) + assert.Equal(t, parsedURL.Scheme, "https") + assert.Equal(t, parsedURL.Host, TestEndpoint) + + params := parsedURL.Query() + + assert.Equal(t, params.Get("Action"), "kafka-cluster:Connect") + assert.Equal(t, params.Get("X-Amz-Algorithm"), "AWS4-HMAC-SHA256") + assert.Equal(t, params.Get("X-Amz-Expires"), "900") + assert.Equal(t, params.Get("X-Amz-Security-Token"), "") + assert.Equal(t, params.Get("X-Amz-SignedHeaders"), "host") + credential := params.Get("X-Amz-Credential") + splitCredential := strings.Split(credential, "/") + assert.Equal(t, splitCredential[0], mockCreds.AccessKeyID) + assert.Equal(t, splitCredential[2], TestRegion) + assert.Equal(t, splitCredential[3], "kafka-cluster") + assert.Equal(t, splitCredential[4], "aws4_request") + date, err := time.Parse("20060102T150405Z", params.Get("X-Amz-Date")) + assert.NoError(t, err) + assert.True(t, date.Before(time.Now().UTC())) + assert.True(t, strings.HasPrefix(params.Get(UserAgentKey), "aws-msk-iam-sasl-signer-go/")) + + signingTimeMs := date.UnixNano() / int64(time.Millisecond) + expiryDurationSeconds, err := strconv.ParseInt(params.Get("X-Amz-Expires"), 10, 64) + assert.NoError(t, err) + expiryDurationMs := expiryDurationSeconds * 1000 + assert.Equal(t, expiryMs, signingTimeMs+expiryDurationMs) + + currentMillis := time.Now().UnixNano() / int64(time.Millisecond) + assert.True(t, expiryMs > currentMillis) +} + +func TestGenerateAuthTokenWithFailingCredentialsProvider(t *testing.T) { + mockCredentialsProvider := aws.AnonymousCredentials{} + + token, expiryMs, err := GenerateAuthTokenFromCredentialsProvider(Ctx, TestRegion, mockCredentialsProvider) + + assert.Error(t, err) + assert.NotNil(t, token) + assert.Equal(t, int64(0), expiryMs) +} diff --git a/signer/version.go b/signer/version.go new file mode 100644 index 0000000..be15f2d --- /dev/null +++ b/signer/version.go @@ -0,0 +1,3 @@ +package signer + +const version = "0.0.1"