Skip to content

Commit

Permalink
feat(core): enhance plugin management
Browse files Browse the repository at this point in the history 
Changes:
* add new interface PluginManager
* add new CLI for un-installing plugins
* refactor service for downloading plugins
* refactor PluginController
  • Loading branch information
@fhussonnois
fhussonnois committed Feb 4, 2025
1 parent 42ad09d commit bb7cc38
Show file tree
Hide file tree
Showing 21 changed files with 1,182 additions and 327 deletions.
16 changes: 11 additions & 5 deletions cli/src/main/java/io/kestra/cli/AbstractCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
import io.kestra.cli.commands.servers.ServerCommandInterface;
import io.kestra.cli.services.StartupHookInterface;
import io.kestra.core.contexts.KestraContext;
import io.kestra.core.plugins.PluginManager;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.webserver.services.FlowAutoLoaderService;
import io.micronaut.context.ApplicationContext;
import io.micronaut.context.env.yaml.YamlPropertySourceLoader;
import io.micronaut.core.annotation.Introspected;
import io.micronaut.management.endpoint.EndpointDefaultConfiguration;
import io.micronaut.runtime.server.EmbeddedServer;
import jakarta.inject.Provider;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.utils.URIBuilder;
import io.kestra.core.utils.Rethrow;
Expand Down Expand Up @@ -49,6 +51,12 @@ abstract public class AbstractCommand implements Callable<Integer> {
@Inject
private io.kestra.core.utils.VersionProvider versionProvider;

@Inject
protected Provider<PluginRegistry> pluginRegistryProvider;

@Inject
protected Provider<PluginManager> pluginManagerProvider;

private PluginRegistry pluginRegistry;

@CommandLine.Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER)
Expand Down Expand Up @@ -84,8 +92,10 @@ public Integer call() throws Exception {
}

if (this.pluginsPath != null && loadExternalPlugins()) {
pluginRegistry = pluginRegistry();
pluginRegistry = pluginRegistryProvider.get();
pluginRegistry.registerIfAbsent(pluginsPath);

pluginManagerProvider.get(); // This will trigger initialization of the plugin-manager
}

startWebserver();
Expand All @@ -102,10 +112,6 @@ protected boolean loadExternalPlugins() {
return true;
}

protected PluginRegistry pluginRegistry() {
return KestraContext.getContext().getPluginRegistry(); // Lazy init
}

private static String message(String message, Object... format) {
return CommandLine.Help.Ansi.AUTO.string(
format.length == 0 ? message : MessageFormat.format(message, format)
Expand Down
5 changes: 4 additions & 1 deletion cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.google.common.io.Files;
import io.kestra.cli.AbstractCommand;
import io.kestra.core.docs.DocumentationGenerator;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.core.plugins.RegisteredPlugin;
import io.kestra.core.serializers.JacksonMapper;
import io.micronaut.context.ApplicationContext;
Expand Down Expand Up @@ -42,8 +43,10 @@ public Integer call() throws Exception {
super.call();
DocumentationGenerator documentationGenerator = applicationContext.getBean(DocumentationGenerator.class);

List<RegisteredPlugin> plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins();
PluginRegistry registry = pluginRegistryProvider.get();
List<RegisteredPlugin> plugins = core ? registry.plugins() : registry.externalPlugins();
boolean hasFailures = false;

for (RegisteredPlugin registeredPlugin : plugins) {
try {
documentationGenerator
Expand Down
108 changes: 54 additions & 54 deletions cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,98 +1,98 @@
package io.kestra.cli.commands.plugins;

import org.apache.commons.io.FilenameUtils;
import io.kestra.core.contexts.MavenPluginRepositoryConfig;
import io.kestra.core.plugins.LocalPluginManager;
import io.kestra.core.plugins.MavenPluginRepository;
import io.kestra.core.plugins.PluginArtifact;
import io.kestra.core.plugins.PluginManager;
import io.micronaut.http.uri.UriBuilder;
import io.kestra.cli.AbstractCommand;
import io.kestra.cli.plugins.PluginDownloader;
import io.kestra.cli.plugins.RepositoryConfig;
import io.kestra.core.utils.IdUtils;
import org.apache.http.client.utils.URIBuilder;
import jakarta.inject.Provider;
import picocli.CommandLine;

import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import jakarta.inject.Inject;
import picocli.CommandLine.Command;
import picocli.CommandLine.Parameters;
import picocli.CommandLine.Option;
import picocli.CommandLine.Spec;

import static io.kestra.core.utils.Rethrow.throwConsumer;

@CommandLine.Command(
@Command(
name = "install",
description = "Install plugins"
)
public class PluginInstallCommand extends AbstractCommand {
@CommandLine.Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.")

@Parameters(index = "0..*", description = "Plugins to install. Represented as Maven artifact coordinates.")
List<String> dependencies = new ArrayList<>();

@CommandLine.Option(names = {"--repositories"}, description = "URL to additional Maven repositories")
@Option(names = {"--repositories"}, description = "URL to additional Maven repositories")
private URI[] repositories;

@CommandLine.Spec
@Spec
CommandLine.Model.CommandSpec spec;

@Inject
private PluginDownloader pluginDownloader;
Provider<MavenPluginRepository> mavenPluginRepositoryProvider;

@Override
public Integer call() throws Exception {
super.call();

if (this.pluginsPath == null) {
throw new CommandLine.ParameterException(this.spec.commandLine(), "Missing required options '--plugins' " +
"or environment variable 'KESTRA_PLUGINS_PATH"
);
}

if (!pluginsPath.toFile().exists()) {
if (!pluginsPath.toFile().mkdir()) {
throw new RuntimeException("Cannot create directory: " + pluginsPath.toFile().getAbsolutePath());
}
}

List<MavenPluginRepositoryConfig> repositoryConfigs = List.of();
if (repositories != null) {
Arrays.stream(repositories)
.forEach(throwConsumer(s -> {
URIBuilder uriBuilder = new URIBuilder(s);

RepositoryConfig.RepositoryConfigBuilder builder = RepositoryConfig.builder()
repositoryConfigs = Arrays.stream(repositories)
.map(uri -> {
MavenPluginRepositoryConfig.MavenPluginRepositoryConfigBuilder builder = MavenPluginRepositoryConfig
.builder()
.id(IdUtils.create());

if (uriBuilder.getUserInfo() != null) {
int index = uriBuilder.getUserInfo().indexOf(":");

builder.basicAuth(new RepositoryConfig.BasicAuth(
uriBuilder.getUserInfo().substring(0, index),
uriBuilder.getUserInfo().substring(index + 1)
String userInfo = uri.getUserInfo();
if (userInfo != null) {
String[] userInfoParts = userInfo.split(":");
builder = builder.basicAuth(new MavenPluginRepositoryConfig.BasicAuth(
userInfoParts[0],
userInfoParts[1]
));

uriBuilder.setUserInfo(null);
}
builder.url(UriBuilder.of(uri).userInfo(null).build().toString());
return builder.build();
}).toList();
}

builder.url(uriBuilder.build().toString());

pluginDownloader.addRepository(builder.build());
}));
final List<PluginArtifact> pluginArtifacts;
try {
pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList();
} catch (IllegalArgumentException e) {
stdErr(e.getMessage());
return CommandLine.ExitCode.USAGE;
}

List<URL> resolveUrl = pluginDownloader.resolve(dependencies);
stdOut("Resolved Plugin(s) with {0}", resolveUrl);
final PluginManager pluginManager;

for (URL url: resolveUrl) {
Files.copy(
Paths.get(url.toURI()),
Paths.get(pluginsPath.toString(), FilenameUtils.getName(url.toString())),
StandardCopyOption.REPLACE_EXISTING
);
// If a PLUGIN_PATH is provided, then use the LocalPluginManager
if (pluginsPath != null) {
pluginManager = new LocalPluginManager(mavenPluginRepositoryProvider.get());
} else {
// Otherwise, we delegate to the configured plugin-manager.
pluginManager = this.pluginManagerProvider.get();
}

stdOut("Successfully installed plugins {0} into {1}", dependencies, pluginsPath);
List<PluginArtifact> installed = pluginManager.install(
pluginArtifacts,
repositoryConfigs,
false,
pluginsPath
);

return 0;
List<URI> uris = installed.stream().map(PluginArtifact::uri).toList();
stdOut("Successfully installed plugins {0} into {1}", dependencies, uris);
return CommandLine.ExitCode.OK;
}

@Override
Expand Down
5 changes: 4 additions & 1 deletion cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.cli.commands.plugins;

import io.kestra.cli.AbstractCommand;
import io.kestra.core.plugins.PluginRegistry;
import io.kestra.core.plugins.RegisteredPlugin;
import picocli.CommandLine;

Expand All @@ -27,7 +28,9 @@ public Integer call() throws Exception {
);
}

List<RegisteredPlugin> plugins = core ? pluginRegistry().plugins() : pluginRegistry().externalPlugins();
PluginRegistry registry = pluginRegistryProvider.get();
List<RegisteredPlugin> plugins = core ? registry.plugins() : registry.externalPlugins();

plugins.forEach(registeredPlugin -> stdOut(registeredPlugin.toString()));

return 0;
Expand Down
69 changes: 69 additions & 0 deletions cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package io.kestra.cli.commands.plugins;

import io.kestra.cli.AbstractCommand;
import io.kestra.core.plugins.LocalPluginManager;
import io.kestra.core.plugins.MavenPluginRepository;
import io.kestra.core.plugins.PluginArtifact;
import io.kestra.core.plugins.PluginManager;
import jakarta.inject.Inject;
import jakarta.inject.Provider;
import picocli.CommandLine;
import picocli.CommandLine.Parameters;
import picocli.CommandLine.Spec;

import java.net.URI;
import java.util.ArrayList;
import java.util.List;

@CommandLine.Command(
name = "uninstall",
description = "uninstall a plugin"
)
public class PluginUninstallCommand extends AbstractCommand {
@Parameters(index = "0..*", description = "the plugins to uninstall")
List<String> dependencies = new ArrayList<>();

@Spec
CommandLine.Model.CommandSpec spec;

@Inject
Provider<MavenPluginRepository> mavenPluginRepositoryProvider;

@Override
public Integer call() throws Exception {
super.call();

List<PluginArtifact> pluginArtifacts;
try {
pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList();
} catch (IllegalArgumentException e) {
stdErr(e.getMessage());
return CommandLine.ExitCode.USAGE;
}

final PluginManager pluginManager;

// If a PLUGIN_PATH is provided, then use the LocalPluginManager
if (pluginsPath != null) {
pluginManager = new LocalPluginManager(mavenPluginRepositoryProvider.get());
} else {
// Otherwise, we delegate to the configured plugin-manager.
pluginManager = this.pluginManagerProvider.get();
}

List<PluginArtifact> uninstalled = pluginManager.uninstall(
pluginArtifacts,
false,
pluginsPath
);

List<URI> uris = uninstalled.stream().map(PluginArtifact::uri).toList();
stdOut("Successfully uninstalled plugins {0} from {1}", dependencies, uris);
return CommandLine.ExitCode.OK;
}

@Override
protected boolean loadExternalPlugins() {
return false;
}
}
Loading

0 comments on commit bb7cc38

Please sign in to comment.