Skip to content

Commit

Permalink
Merge a16059d into 162a193
Browse files Browse the repository at this point in the history
  • Loading branch information
rm3l authored Feb 7, 2023
2 parents 162a193 + a16059d commit 853ed43
Show file tree
Hide file tree
Showing 9 changed files with 213 additions and 30 deletions.
14 changes: 11 additions & 3 deletions docs/website/docs/user-guides/advanced/pushing-specific-files.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,18 @@
title: Only Push Specific Files
sidebar_position: 2
---
`odo` uses the `odo.dev.push.path` related attribute from the devfile's run commands to push only the specified files and folders to the component.
`odo` uses the `dev.odo.push.path` related attribute from the devfile's run commands to push only the specified files and folders to the component.

The format of the attribute is `"odo.dev.push.path:<local_relative_path>": "<remote_relative_path>"`. We can mention multiple such attributes in the run command's `attributes` section.
The format of the attribute is `"dev.odo.push.path:<local_relative_path>": "<remote_relative_path>"`. We can mention multiple such attributes in the run command's `attributes` section.

```yaml
commands:
- id: dev-run
# highlight-start
attributes:
"dev.odo.push.path:target/quarkus-app": "remote-target/quarkus-app"
"dev.odo.push.path:README.txt": "docs/README.txt"
# highlight-end
exec:
component: tools
commandLine: "java -jar remote-target/quarkus-app/quarkus-run.jar -Dquarkus.http.host=0.0.0.0"
Expand All @@ -21,6 +23,11 @@ commands:
isDefault: true
workingDir: $PROJECTS_ROOT
- id: dev-debug
# highlight-start
attributes:
"dev.odo.push.path:target/quarkus-app": "remote-target/quarkus-app"
"dev.odo.push.path:README.txt": "docs/README.txt"
# highlight-end
exec:
component: tools
commandLine: "java -Xdebug -Xrunjdwp:server=y,transport=dt_socket,address=${DEBUG_PORT},suspend=n -jar remote-target/quarkus-app/quarkus-run.jar -Dquarkus.http.host=0.0.0.0"
Expand All @@ -31,4 +38,5 @@ commands:
workingDir: $PROJECTS_ROOT
```
In the above example the contents of the `quarkus-app` folder, which is inside the `target` folder, will be pushed to the remote location of `remote-target/quarkus-app` and the file `README.txt` will be pushed to `doc/README.txt`. The local path is relative to the component's local folder. The remote location is relative to the folder containing the component's source code inside the container.
In the above example the contents of the `quarkus-app` folder, which is inside the `target` folder, will be pushed to the remote location of `remote-target/quarkus-app` and the file `README.txt` will be pushed to `doc/README.txt`.
The local path is relative to the component's local folder. The remote location is relative to the folder containing the component's source code inside the container.
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
---
title: Using the odo.dev.push.path related attribute
title: Using the dev.odo.push.path related attribute
sidebar_position: 4
---
`odo` uses the `odo.dev.push.path` related attribute from the devfile's run commands to push only the specified files and folders to the component.
`odo` uses the `dev.odo.push.path` related attribute from the devfile's run commands to push only the specified files and folders to the component.

The format of the attribute is `"odo.dev.push.path:<local_relative_path>": "<remote_relative_path>"`. We can mention multiple such attributes in the run command's `attributes` section.
The format of the attribute is `"dev.odo.push.path:<local_relative_path>": "<remote_relative_path>"`. We can mention multiple such attributes in the run command's `attributes` section.

```yaml
commands:
- id: dev-run
# highlight-start
attributes:
"dev.odo.push.path:target/quarkus-app": "remote-target/quarkus-app"
"dev.odo.push.path:README.txt": "docs/README.txt"
# highlight-end
exec:
component: tools
commandLine: "java -jar remote-target/quarkus-app/quarkus-run.jar -Dquarkus.http.host=0.0.0.0"
Expand Down
16 changes: 15 additions & 1 deletion pkg/dev/podmandev/podmandev.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"path/filepath"
"strings"

devfilev1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/library/v2/pkg/devfile/parser"
"k8s.io/klog"

Expand All @@ -18,6 +19,7 @@ import (
"github.com/redhat-developer/odo/pkg/devfile/adapters"
"github.com/redhat-developer/odo/pkg/devfile/location"
"github.com/redhat-developer/odo/pkg/exec"
"github.com/redhat-developer/odo/pkg/libdevfile"
"github.com/redhat-developer/odo/pkg/log"
odocontext "github.com/redhat-developer/odo/pkg/odo/context"
"github.com/redhat-developer/odo/pkg/podman"
Expand Down Expand Up @@ -113,6 +115,7 @@ func (o *DevClient) Start(
// syncFiles syncs the local source files in path into the pod's source volume
func (o *DevClient) syncFiles(ctx context.Context, options dev.StartOptions, pod *corev1.Pod, path string) (bool, error) {
var (
devfileObj = odocontext.GetDevfileObj(ctx)
componentName = odocontext.GetComponentName(ctx)
)

Expand All @@ -128,6 +131,17 @@ func (o *DevClient) syncFiles(ctx context.Context, options dev.StartOptions, pod
SyncFolder: syncFolder,
}

cmdKind := devfilev1.RunCommandGroupKind
cmdName := options.RunCommand
if options.Debug {
cmdKind = devfilev1.DebugCommandGroupKind
cmdName = options.DebugCommand
}
devfileCmd, err := libdevfile.ValidateAndGetCommand(*devfileObj, cmdName, cmdKind)
if err != nil {
return false, err
}

syncParams := sync.SyncParameters{
Path: path,
WatchFiles: nil,
Expand All @@ -137,7 +151,7 @@ func (o *DevClient) syncFiles(ctx context.Context, options dev.StartOptions, pod

CompInfo: compInfo,
ForcePush: true,
Files: map[string]string{}, // ??? TODO
Files: adapters.GetSyncFilesFromAttributes(devfileCmd),
}
execRequired, err := o.syncClient.SyncFiles(syncParams)
if err != nil {
Expand Down
23 changes: 23 additions & 0 deletions pkg/devfile/adapters/attributes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package adapters

import (
"path/filepath"
"strings"

"github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
)

const _devPushPathAttributePrefix = "dev.odo.push.path:"

// GetSyncFilesFromAttributes gets the target files and folders along with their respective remote destination from the devfile.
// It uses the "dev.odo.push.path:" attribute prefix, if any, in the specified command.
func GetSyncFilesFromAttributes(command v1alpha2.Command) map[string]string {
syncMap := make(map[string]string)
for key, value := range command.Attributes.Strings(nil) {
if strings.HasPrefix(key, _devPushPathAttributePrefix) {
localValue := strings.ReplaceAll(key, _devPushPathAttributePrefix, "")
syncMap[filepath.Clean(localValue)] = filepath.ToSlash(filepath.Clean(value))
}
}
return syncMap
}
91 changes: 91 additions & 0 deletions pkg/devfile/adapters/attributes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package adapters

import (
"testing"

"github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
"github.com/devfile/api/v2/pkg/attributes"
"github.com/google/go-cmp/cmp"
)

func TestGetSyncFilesFromAttributes(t *testing.T) {
type args struct {
command v1alpha2.Command
}
tests := []struct {
name string
args args
want map[string]string
}{
{
name: "no attributes",
args: args{
command: v1alpha2.Command{},
},
want: make(map[string]string),
},
{
name: "no matching attributes",
args: args{
command: v1alpha2.Command{
Attributes: attributes.Attributes{}.FromStringMap(map[string]string{
"some-custom-attribute-key": "some-value",
"another-custom-attribute-key": "some-value",
}),
},
},
want: make(map[string]string),
},
{
name: "dev.odo.push.path attribute key",
args: args{
command: v1alpha2.Command{
Attributes: attributes.Attributes{}.FromStringMap(map[string]string{
"dev.odo.push.path": "some-value",
}),
},
},
want: make(map[string]string),
},
{
name: "attribute with only matching prefix as key",
args: args{
command: v1alpha2.Command{
Attributes: attributes.Attributes{}.FromStringMap(map[string]string{
_devPushPathAttributePrefix: "server/",
}),
},
},
want: map[string]string{
".": "server",
},
},
{
name: "multiple matching attributes",
args: args{
command: v1alpha2.Command{
Attributes: attributes.Attributes{}.FromStringMap(map[string]string{
"some-custom-attribute-key": "some-value",
_devPushPathAttributePrefix + "server.js": "server/server.js",
"some-other-custom-attribute-key": "some-value",
_devPushPathAttributePrefix + "some-path/README": "another/nested/path/README.md",
_devPushPathAttributePrefix + "random-file.txt": "/tmp/rand-file.txt",
}),
},
},
want: map[string]string{
"server.js": "server/server.js",
"some-path/README": "another/nested/path/README.md",
"random-file.txt": "/tmp/rand-file.txt",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GetSyncFilesFromAttributes(tt.args.command)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("GetSyncFilesFromAttributes() mismatch (-want +got):\n%s", diff)
}
})
}
}
26 changes: 7 additions & 19 deletions pkg/devfile/adapters/kubernetes/component/adapter.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"path/filepath"
"reflect"
"strings"
sync2 "sync"
Expand Down Expand Up @@ -234,6 +233,12 @@ func (a Adapter) Push(ctx context.Context, parameters adapters.PushParameters, c
SyncFolder: syncFolder,
}

cmdKind := devfilev1.RunCommandGroupKind
if parameters.Debug {
cmdKind = devfilev1.DebugCommandGroupKind
}
devfileCmd := pushDevfileCommands[cmdKind]

syncParams := sync.SyncParameters{
Path: parameters.Path,
WatchFiles: parameters.WatchFiles,
Expand All @@ -243,7 +248,7 @@ func (a Adapter) Push(ctx context.Context, parameters adapters.PushParameters, c

CompInfo: compInfo,
ForcePush: !deploymentExists || podChanged,
Files: getSyncFilesFromAttributes(pushDevfileCommands),
Files: adapters.GetSyncFilesFromAttributes(devfileCmd),
}

execRequired, err := a.syncClient.SyncFiles(syncParams)
Expand All @@ -264,10 +269,8 @@ func (a Adapter) Push(ctx context.Context, parameters adapters.PushParameters, c
}
componentStatus.PostStartEventsDone = true

cmdKind := devfilev1.RunCommandGroupKind
cmdName := parameters.DevfileRunCmd
if parameters.Debug {
cmdKind = devfilev1.DebugCommandGroupKind
cmdName = parameters.DevfileDebugCmd
}

Expand Down Expand Up @@ -758,18 +761,3 @@ func (a Adapter) deleteServiceBindingSecrets(serviceBindingSecretsToRemove []uns

// PushCommandsMap stores the commands to be executed as per their types.
type PushCommandsMap map[devfilev1.CommandGroupKind]devfilev1.Command

// getSyncFilesFromAttributes gets the target files and folders along with their respective remote destination from the devfile
// it uses the "dev.odo.push.path" attribute in the run command
func getSyncFilesFromAttributes(commandsMap PushCommandsMap) map[string]string {
syncMap := make(map[string]string)
if value, ok := commandsMap[devfilev1.RunCommandGroupKind]; ok {
for key, value := range value.Attributes.Strings(nil) {
if strings.HasPrefix(key, "dev.odo.push.path:") {
localValue := strings.ReplaceAll(key, "dev.odo.push.path:", "")
syncMap[filepath.Clean(localValue)] = filepath.ToSlash(filepath.Clean(value))
}
}
}
return syncMap
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ components:
endpoints:
- name: "3000-tcp"
targetPort: 3000
- name: debug
targetPort: 5858
mountSources: true
commands:
- id: devbuild
Expand Down Expand Up @@ -50,3 +52,18 @@ commands:
workingDir: ${PROJECTS_ROOT}
group:
kind: run
- id: devdebug
attributes:
"dev.odo.push.path:server.js": "server-debug/server.js"
"dev.odo.push.path:test": "server-debug/test"
"dev.odo.push.path:package.json": "package.json"
exec:
component: runtime
commandLine: npm run debug
workingDir: ${PROJECTS_ROOT}
env:
- name: DEBUG_PORT_PROJECT
value: "5858"
group:
kind: debug
isDefault: true
44 changes: 44 additions & 0 deletions tests/integration/cmd_dev_debug_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"k8s.io/utils/pointer"

"github.com/redhat-developer/odo/pkg/labels"
"github.com/redhat-developer/odo/tests/helper"

. "github.com/onsi/ginkgo/v2"
Expand Down Expand Up @@ -81,6 +82,49 @@ var _ = Describe("odo dev debug command tests", func() {
})
}

for _, podman := range []bool{false, true} {
podman := podman
When("creating nodejs component, doing odo dev and run command has dev.odo.push.path attribute", helper.LabelPodmanIf(podman, func() {
var session helper.DevSession
var devStarted bool
BeforeEach(func() {
helper.Cmd("odo", "init", "--name", cmpName, "--devfile-path",
helper.GetExamplePath("source", "devfiles", "nodejs", "devfile-with-remote-attributes.yaml")).ShouldPass()
helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context)

// create a folder and file which shouldn't be pushed
helper.MakeDir(filepath.Join(commonVar.Context, "views"))
_, _ = helper.CreateSimpleFile(filepath.Join(commonVar.Context, "views"), "view", ".html")

helper.ReplaceString("package.json", "node server.js", "node server-debug/server.js")
var err error
session, _, _, _, err = helper.StartDevMode(helper.DevSessionOpts{
RunOnPodman: podman,
CmdlineArgs: []string{"--debug"},
})
Expect(err).ToNot(HaveOccurred())
devStarted = true
})
AfterEach(func() {
if devStarted {
session.Stop()
session.WaitEnd()
}
})

It("should sync only the mentioned files at the appropriate remote destination", func() {
component := helper.NewComponent(cmpName, "app", labels.ComponentDevMode, commonVar.Project, commonVar.CliRunner)
stdOut, _ := component.Exec("runtime", []string{"ls", "-lai", "/projects"}, pointer.Bool(true))

helper.MatchAllInOutput(stdOut, []string{"package.json", "server-debug"})
helper.DontMatchAllInOutput(stdOut, []string{"test", "views", "devfile.yaml"})

stdOut, _ = component.Exec("runtime", []string{"ls", "-lai", "/projects/server-debug"}, pointer.Bool(true))
helper.MatchAllInOutput(stdOut, []string{"server.js", "test"})
})
}))
}

for _, devfileHandlerCtx := range []struct {
name string
sourceHandler func(path string, originalCmpName string)
Expand Down
4 changes: 0 additions & 4 deletions tests/integration/cmd_dev_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2604,13 +2604,9 @@ CMD ["npm", "start"]
for _, podman := range []bool{false, true} {
podman := podman
When("creating nodejs component, doing odo dev and run command has dev.odo.push.path attribute", helper.LabelPodmanIf(podman, func() {
// TODO Not implemented yet on Podman
var session helper.DevSession
var devStarted bool
BeforeEach(func() {
if podman {
Skip("not implemented yet on Podman - see #6492")
}
helper.Cmd("odo", "init", "--name", cmpName, "--devfile-path",
helper.GetExamplePath("source", "devfiles", "nodejs", "devfile-with-remote-attributes.yaml")).ShouldPass()
helper.CopyExample(filepath.Join("source", "devfiles", "nodejs", "project"), commonVar.Context)
Expand Down

0 comments on commit 853ed43

Please sign in to comment.