Skip to content

Check and generate multiple readme files in one package #282

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kaiyan-sheng merged 23 commits into from
Mar 23, 2021
Merged

Check and generate multiple readme files in one package #282

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
12a17ea
Check and generate multiple README.md files in one package
Mar 9, 2021
a3886c0
Merge branch 'master' into additional_readmes
ycombinator Mar 9, 2021
ae666b8
move ReadDir into IsReadmeUpToDate and UpdateReadme
Mar 10, 2021
0981abe
Merge branch 'master' of github.com:elastic/elastic-package into addi…
Mar 10, 2021
d57e8ba
Merge branch 'additional_readmes' of github.com:kaiyan-sheng/elastic-…
Mar 10, 2021
7ef4eb5
Merge branch 'master' into additional_readmes
ycombinator Mar 10, 2021
3e762da
Merge branch 'master' into additional_readmes
Mar 12, 2021
303bb12
Create AreReadmesUpToDate and UpdateReadmes functions
Mar 13, 2021
17afb55
print elastic-package build output in alphabetical order
Mar 13, 2021
4f603f8
list all file names that need to be updated
Mar 13, 2021
5898df2
make _dev/build/docs not required
Mar 15, 2021
6cf1ce4
Merge branch 'master' into additional_readmes
mtojek Mar 15, 2021
e5983ce
check os.IsNotExist and use fileName instead
Mar 16, 2021
380019c
Merge branch 'additional_readmes' of github.com:kaiyan-sheng/elastic-…
Mar 16, 2021
41c2ff2
fix error message
Mar 16, 2021
9d9039c
Merge branch 'master' into additional_readmes
mtojek Mar 16, 2021
9a85bb5
use cmd.Printf for error messages
Mar 18, 2021
da89e55
Merge branch 'additional_readmes' of github.com:kaiyan-sheng/elastic-…
Mar 18, 2021
cb3e976
fix error messages
Mar 18, 2021
9ff5e57
add ignore_missing
Mar 18, 2021
1673749
rerun elastic-package test pipeline for aws
Mar 18, 2021
d24311c
test with adding package level vars
Mar 22, 2021
17506c5
check input is nil before merging vars
Mar 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions cmd/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,15 @@
package cmd

import (
"io/ioutil"
"path/filepath"

"github.com/pkg/errors"
"github.com/spf13/cobra"

"github.com/elastic/elastic-package/internal/builder"
"github.com/elastic/elastic-package/internal/docs"
"github.com/elastic/elastic-package/internal/packages"
)

const buildLongDescription = `Use this command to build a package. Currently it supports only the "integration" package type.
Expand All @@ -36,19 +40,35 @@ func setupBuildCommand() *cobra.Command {
func buildCommandAction(cmd *cobra.Command, args []string) error {
cmd.Println("Build the package")

target, err := docs.UpdateReadme()
if err != nil {
return errors.Wrapf(err, "updating %s file failed", docs.ReadmeFile)
packageRootPath, found, err := packages.FindPackageRoot()
if !found {
return errors.New("package root not found")
}
if target != "" {
cmd.Printf("%s file rendered: %s\n", docs.ReadmeFile, target)
if err != nil {
return errors.Wrap(err, "locating package root failed")
}

target, err = builder.BuildPackage()
readmeFiles, err := ioutil.ReadDir(filepath.Join(packageRootPath, "_dev", "build", "docs"))
Comment thread
mtojek marked this conversation as resolved.
Outdated
Comment thread
kaiyan-sheng marked this conversation as resolved.
Outdated
if err != nil {
return errors.Wrap(err, "building package failed")
return errors.Wrapf(err, "failed to return a list of directory entries from %s", packageRootPath)
}

for _, readme := range readmeFiles {
fileName := readme.Name()
target, err := docs.UpdateReadme(fileName)
if err != nil {
return errors.Wrapf(err, "updating %s file failed", fileName)
}
if target != "" {

@mtojek mtojek Mar 15, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the case that the target is empty?

@kaiyan-sheng kaiyan-sheng Mar 16, 2021

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It will be empty when README file is static and can't be generated from the template file. This came from generateReadme(fileName, packageRoot) function.

@mtojek mtojek Mar 16, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I see! It might be cleaner to skip this file on the lower layer (in the docs.UpdateReadmes()), so "" (empty string) won't leak here.

cmd.Printf("%s file rendered: %s\n", fileName, target)
}

target, err = builder.BuildPackage()
if err != nil {
return errors.Wrap(err, "building package failed")
}
cmd.Printf("Package built: %s\n", target)
}
cmd.Printf("Package built: %s\n", target)

cmd.Println("Done")
return nil
Expand Down
21 changes: 15 additions & 6 deletions cmd/lint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@ package cmd

import (
"fmt"
"io/ioutil"
"path/filepath"

"github.com/pkg/errors"
"github.com/spf13/cobra"

"github.com/elastic/package-spec/code/go/pkg/validator"

"github.com/elastic/elastic-package/internal/docs"
"github.com/elastic/elastic-package/internal/packages"
"github.com/elastic/package-spec/code/go/pkg/validator"
)

const lintLongDescription = `Use this command to validate the contents of a package using the package specification (see: https://github.com/elastic/package-spec).
Expand Down Expand Up @@ -44,12 +45,20 @@ func lintCommandAction(cmd *cobra.Command, args []string) error {
return errors.Wrap(err, "locating package root failed")
}

ok, err := docs.IsReadmeUpToDate()
readmeFiles, err := ioutil.ReadDir(filepath.Join(packageRootPath, "_dev", "build", "docs"))
Comment thread
kaiyan-sheng marked this conversation as resolved.
Outdated
if err != nil {
return errors.Wrapf(err, "can't check if %s file is up-to-date", docs.ReadmeFile)
return errors.Wrapf(err, "failed to return a list of directory entries from %s", packageRootPath)
}
if !ok {
return fmt.Errorf("%s file is outdated. Rebuild the package with 'elastic-package build'", docs.ReadmeFile)

for _, readme := range readmeFiles {
filename := readme.Name()
ok, err := docs.IsReadmeUpToDate(filename)
if err != nil {
return errors.Wrapf(err, "can't check if %s file is up-to-date", filename)
}
if !ok {
return fmt.Errorf("%s file is outdated. Rebuild the package with 'elastic-package build'", filename)
}
}

err = validator.ValidateFromPath(packageRootPath)
Expand Down
63 changes: 30 additions & 33 deletions internal/docs/readme.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,27 +17,24 @@ import (
"github.com/elastic/elastic-package/internal/packages"
)

// ReadmeFile for the Elastic package
const ReadmeFile = "README.md"

// IsReadmeUpToDate function checks if the README file is up-to-date.
func IsReadmeUpToDate() (bool, error) {
logger.Debugf("Check if %s is up-to-date", ReadmeFile)
func IsReadmeUpToDate(readmeFileName string) (bool, error) {
Comment thread
kaiyan-sheng marked this conversation as resolved.
Outdated
logger.Debugf("Check if %s is up-to-date", readmeFileName)

packageRoot, err := packages.MustFindPackageRoot()
if err != nil {
return false, errors.Wrap(err, "package root not found")
}

rendered, shouldBeRendered, err := generateReadme(packageRoot)
rendered, shouldBeRendered, err := generateReadme(packageRoot, readmeFileName)
if err != nil {
return false, err
}
if !shouldBeRendered {
return true, nil // README file is static and doesn't use template.
}

existing, found, err := readReadme(packageRoot)
existing, found, err := readReadme(packageRoot, readmeFileName)
if err != nil {
return false, errors.Wrap(err, "reading README file failed")
}
Expand All @@ -49,64 +46,64 @@ func IsReadmeUpToDate() (bool, error) {

// UpdateReadme function updates the README file using ą defined template file. The function doesn't perform any action
// if the template file is not present.
func UpdateReadme() (string, error) {
logger.Debugf("Update the %s file", ReadmeFile)
func UpdateReadme(readmeFileName string) (string, error) {
logger.Debugf("Update the %s file", readmeFileName)

packageRoot, err := packages.MustFindPackageRoot()
if err != nil {
return "", errors.Wrap(err, "package root not found")
}

rendered, shouldBeRendered, err := generateReadme(packageRoot)
rendered, shouldBeRendered, err := generateReadme(packageRoot, readmeFileName)
if err != nil {
return "", err
}
if !shouldBeRendered {
return "", nil
}

target, err := writeReadme(packageRoot, rendered)
target, err := writeReadme(readmeFileName, packageRoot, rendered)
if err != nil {
return "", errors.Wrapf(err, "writing %s file failed", ReadmeFile)
return "", errors.Wrapf(err, "writing %s file failed", readmeFileName)
}
return target, nil
}

func generateReadme(packageRoot string) ([]byte, bool, error) {
logger.Debugf("Generate %s file (package: %s)", ReadmeFile, packageRoot)
templatePath, found, err := findReadmeTemplatePath(packageRoot)
func generateReadme(packageRoot string, readmeFileName string) ([]byte, bool, error) {
logger.Debugf("Generate %s file (package: %s)", readmeFileName, packageRoot)
templatePath, found, err := findReadmeTemplatePath(packageRoot, readmeFileName)
if err != nil {
return nil, false, errors.Wrapf(err, "can't locate %s template file", ReadmeFile)
return nil, false, errors.Wrapf(err, "can't locate %s template file", readmeFileName)
}
if !found {
logger.Debug("README file is static, can't be generated from the template file")
return nil, false, nil
}
logger.Debugf("Template file for %s found: %s", ReadmeFile, templatePath)
logger.Debugf("Template file for %s found: %s", readmeFileName, templatePath)

rendered, err := renderReadme(packageRoot, templatePath)
rendered, err := renderReadme(readmeFileName, packageRoot, templatePath)
if err != nil {
return nil, true, errors.Wrap(err, "rendering Readme failed")
}
return rendered, true, nil
}

func findReadmeTemplatePath(packageRoot string) (string, bool, error) {
templatePath := filepath.Join(packageRoot, "_dev", "build", "docs", ReadmeFile)
func findReadmeTemplatePath(packageRoot, readmeFileName string) (string, bool, error) {
templatePath := filepath.Join(packageRoot, "_dev", "build", "docs", readmeFileName)
_, err := os.Stat(templatePath)
if err != nil && os.IsNotExist(err) {
return "", false, nil // README.md file not found
}
if err != nil {
return "", false, errors.Wrapf(err, "can't located the %s file", ReadmeFile)
return "", false, errors.Wrapf(err, "can't located the %s file", readmeFileName)
}
return templatePath, true, nil
}

func renderReadme(packageRoot, templatePath string) ([]byte, error) {
logger.Debugf("Render %s file (package: %s, templatePath: %s)", ReadmeFile, packageRoot, templatePath)
func renderReadme(readmeFileName, packageRoot, templatePath string) ([]byte, error) {
logger.Debugf("Render %s file (package: %s, templatePath: %s)", readmeFileName, packageRoot, templatePath)

t := template.New(ReadmeFile)
t := template.New(readmeFileName)
t, err := t.Funcs(template.FuncMap{
"event": func(dataStreamName string) (string, error) {
return renderSampleEvent(packageRoot, dataStreamName)
Expand All @@ -127,10 +124,10 @@ func renderReadme(packageRoot, templatePath string) ([]byte, error) {
return rendered.Bytes(), nil
}

func readReadme(packageRoot string) ([]byte, bool, error) {
logger.Debugf("Read existing %s file (package: %s)", ReadmeFile, packageRoot)
func readReadme(packageRoot, readmeFileName string) ([]byte, bool, error) {
logger.Debugf("Read existing %s file (package: %s)", readmeFileName, packageRoot)

readmePath := filepath.Join(packageRoot, "docs", ReadmeFile)
readmePath := filepath.Join(packageRoot, "docs", readmeFileName)
b, err := ioutil.ReadFile(readmePath)
if err != nil && os.IsNotExist(err) {
return nil, false, nil
Expand All @@ -141,8 +138,8 @@ func readReadme(packageRoot string) ([]byte, bool, error) {
return b, true, err
}

func writeReadme(packageRoot string, content []byte) (string, error) {
logger.Debugf("Write %s file (package: %s)", ReadmeFile, packageRoot)
func writeReadme(readmeFileName, packageRoot string, content []byte) (string, error) {
logger.Debugf("Write %s file (package: %s)", readmeFileName, packageRoot)

docsPath := docsPath(packageRoot)
logger.Debugf("Create directories: %s", docsPath)
Expand All @@ -151,8 +148,8 @@ func writeReadme(packageRoot string, content []byte) (string, error) {
return "", errors.Wrapf(err, "mkdir failed (path: %s)", docsPath)
}

aReadmePath := readmePath(packageRoot)
logger.Debugf("Write %s file to: %s", ReadmeFile, aReadmePath)
aReadmePath := readmePath(packageRoot, readmeFileName)
logger.Debugf("Write %s file to: %s", readmeFileName, aReadmePath)

err = ioutil.WriteFile(aReadmePath, content, 0644)
if err != nil {
Expand All @@ -161,8 +158,8 @@ func writeReadme(packageRoot string, content []byte) (string, error) {
return aReadmePath, nil
}

func readmePath(packageRoot string) string {
return filepath.Join(docsPath(packageRoot), ReadmeFile)
func readmePath(packageRoot, readmeFileName string) string {
return filepath.Join(docsPath(packageRoot), readmeFileName)
}

func docsPath(packageRoot string) string {
Expand Down
Loading