Skip to content

Commit 479d6de

Browse files
pingsutwFuture-Outlier
pingsutw
and
Future-Outlier
authored
Rename agent to connector (#3165)
Signed-off-by: Kevin Su <[email protected]> Signed-off-by: Future-Outlier <[email protected]> Co-authored-by: Future-Outlier <[email protected]>
1 parent 457a6bb commit 479d6de

File tree

85 files changed

+1196
-952
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+1196
-952
lines changed

Diff for: ‎.github/workflows/build_image.yml

+39-1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,44 @@ jobs:
8181
tags: ${{ steps.flyteagent-names.outputs.tags }}
8282
build-args: |
8383
VERSION=${{ github.sha }}
84-
file: ./Dockerfile.agent
84+
file: ./Dockerfile.connector
85+
cache-from: type=gha
86+
cache-to: type=gha,mode=max
87+
88+
build-and-push-flyteconnector-images-manual:
89+
runs-on: ubuntu-latest
90+
steps:
91+
- uses: actions/checkout@v4
92+
with:
93+
fetch-depth: "0"
94+
- name: Set up QEMU
95+
uses: docker/setup-qemu-action@v3
96+
- name: Set up Docker Buildx
97+
id: buildx
98+
uses: docker/setup-buildx-action@v3
99+
- name: Login to GitHub Container Registry
100+
uses: docker/login-action@v3
101+
with:
102+
registry: ghcr.io
103+
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
104+
password: "${{ secrets.FLYTE_BOT_PAT }}"
105+
- name: Prepare Flyte Connector Image Names
106+
id: flyteconnector-names
107+
uses: docker/metadata-action@v3
108+
with:
109+
images: |
110+
ghcr.io/${{ github.repository_owner }}/flyteconnector
111+
tags: |
112+
${{ github.sha }}
113+
- name: Push External Plugin Service Image to GitHub Registry
114+
uses: docker/build-push-action@v2
115+
with:
116+
context: "."
117+
platforms: linux/arm64, linux/amd64
118+
push: true
119+
tags: ${{ steps.flyteconnector-names.outputs.tags }}
120+
build-args: |
121+
VERSION=${{ github.sha }}
122+
file: ./Dockerfile.connector
85123
cache-from: type=gha
86124
cache-to: type=gha,mode=max

Diff for: ‎.github/workflows/pythonpublish.yml

+70-1
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,76 @@ jobs:
235235
tags: ${{ steps.flyteagent-names.outputs.tags }}
236236
build-args: |
237237
VERSION=${{ needs.deploy.outputs.version }}
238-
file: ./Dockerfile.agent
238+
file: ./Dockerfile.connector
239+
cache-from: type=gha
240+
cache-to: type=gha,mode=max
241+
242+
build-and-push-flyteconnector-images:
243+
runs-on: ubuntu-latest
244+
needs: deploy
245+
steps:
246+
- uses: actions/checkout@v4
247+
with:
248+
fetch-depth: "0"
249+
- name: Set up QEMU
250+
uses: docker/setup-qemu-action@v3
251+
- name: Set up Docker Buildx
252+
id: buildx
253+
uses: docker/setup-buildx-action@v3
254+
- name: Login to GitHub Container Registry
255+
if: ${{ github.event_name == 'release' }}
256+
uses: docker/login-action@v3
257+
with:
258+
registry: ghcr.io
259+
username: "${{ secrets.FLYTE_BOT_USERNAME }}"
260+
password: "${{ secrets.FLYTE_BOT_PAT }}"
261+
- name: Prepare Flyte connector Slim Image Names
262+
id: flyteconnector-slim-names
263+
uses: docker/metadata-action@v3
264+
with:
265+
images: |
266+
ghcr.io/${{ github.repository_owner }}/flyteconnector-slim
267+
tags: |
268+
latest
269+
${{ github.sha }}
270+
${{ needs.deploy.outputs.version }}
271+
- name: Prepare Flyte Connector Image Names
272+
id: flyteconnector-names
273+
uses: docker/metadata-action@v3
274+
with:
275+
images: |
276+
ghcr.io/${{ github.repository_owner }}/flyteconnector
277+
tags: |
278+
latest
279+
${{ github.sha }}
280+
${{ needs.deploy.outputs.version }}
281+
- name: Push flyteconnector-slim Image to GitHub Registry
282+
uses: docker/build-push-action@v2
283+
with:
284+
context: "."
285+
platforms: linux/arm64, linux/amd64
286+
target: connector-slim
287+
push: ${{ github.event_name == 'release' }}
288+
tags: ${{ steps.flyteconnector-slim-names.outputs.tags }}
289+
build-args: |
290+
VERSION=${{ needs.deploy.outputs.version }}
291+
file: ./Dockerfile.connector
292+
cache-from: type=gha
293+
cache-to: type=gha,mode=max
294+
- name: Confirm Connector can start
295+
run: |
296+
docker run --rm ghcr.io/${{ github.repository_owner }}/flyteconnector-slim:${{ github.sha }} pyflyte serve connector --port 8000 --timeout 1
297+
- name: Push flyteconnector-all Image to GitHub Registry
298+
uses: docker/build-push-action@v2
299+
with:
300+
context: "."
301+
platforms: linux/arm64, linux/amd64
302+
target: connector-all
303+
push: ${{ github.event_name == 'release' }}
304+
tags: ${{ steps.flyteconnector-names.outputs.tags }}
305+
build-args: |
306+
VERSION=${{ needs.deploy.outputs.version }}
307+
file: ./Dockerfile.connector
239308
cache-from: type=gha
240309
cache-to: type=gha,mode=max
241310

Diff for: ‎Dockerfile.agent renamed to ‎Dockerfile.connector

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.10-slim-bookworm AS agent-slim
1+
FROM python:3.10-slim-bookworm AS connector-slim
22

33
LABEL org.opencontainers.image.authors="Flyte Team <[email protected]>"
44
LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
@@ -8,7 +8,7 @@ ARG VERSION
88
RUN apt-get update && apt-get install build-essential -y \
99
&& pip install uv
1010

11-
RUN uv pip install --system --no-cache-dir -U flytekit[agent]==$VERSION \
11+
RUN uv pip install --system --no-cache-dir -U flytekit[connector]==$VERSION \
1212
flytekitplugins-airflow==$VERSION \
1313
flytekitplugins-bigquery==$VERSION \
1414
flytekitplugins-k8sdataservice==$VERSION \
@@ -21,9 +21,9 @@ RUN uv pip install --system --no-cache-dir -U flytekit[agent]==$VERSION \
2121
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ \
2222
&& :
2323

24-
CMD ["pyflyte", "serve", "agent", "--port", "8000"]
24+
CMD ["pyflyte", "serve", "connector", "--port", "8000"]
2525

26-
FROM agent-slim AS agent-all
26+
FROM connector-slim AS connector-all
2727
ARG VERSION
2828

2929
RUN uv pip install --system --no-cache-dir -U \

Diff for: ‎flytekit/clis/sdk_in_container/serve.py

+79-19
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import importlib
22
import os
33
import sys
4+
import warnings
45
from concurrent import futures
56

67
import grpc
@@ -68,34 +69,93 @@ def agent(_: click.Context, port, prometheus_port, worker, timeout, modules):
6869
"""
6970
import asyncio
7071

72+
warnings.warn("This command is deprecated. Please use `flyte serve connector` instead.", DeprecationWarning)
73+
7174
working_dir = os.getcwd()
7275
if all(os.path.realpath(path) != working_dir for path in sys.path):
7376
sys.path.append(working_dir)
7477
for m in modules:
7578
importlib.import_module(m)
7679

77-
asyncio.run(_start_grpc_server(port, prometheus_port, worker, timeout))
80+
asyncio.run(_start_grpc_server("Agent", port, prometheus_port, worker, timeout))
7881

7982

80-
async def _start_grpc_server(port: int, prometheus_port: int, worker: int, timeout: int):
83+
@serve.command()
84+
@click.option(
85+
"--port",
86+
default="8000",
87+
is_flag=False,
88+
type=int,
89+
help="Grpc port for the connector service",
90+
)
91+
@click.option(
92+
"--prometheus_port",
93+
default="9090",
94+
is_flag=False,
95+
type=int,
96+
help="Prometheus port for the connector service",
97+
)
98+
@click.option(
99+
"--worker",
100+
default="10",
101+
is_flag=False,
102+
type=int,
103+
help="Number of workers for the grpc server",
104+
)
105+
@click.option(
106+
"--timeout",
107+
default=None,
108+
is_flag=False,
109+
type=int,
110+
help="It will wait for the specified number of seconds before shutting down grpc server. It should only be used "
111+
"for testing.",
112+
)
113+
@click.option(
114+
"--modules",
115+
required=False,
116+
multiple=True,
117+
type=str,
118+
help="List of additional files or module that defines the connector",
119+
)
120+
@click.pass_context
121+
def connector(_: click.Context, port, prometheus_port, worker, timeout, modules):
122+
"""
123+
Start a grpc server for the connector service.
124+
"""
125+
import asyncio
126+
127+
working_dir = os.getcwd()
128+
if all(os.path.realpath(path) != working_dir for path in sys.path):
129+
sys.path.append(working_dir)
130+
for m in modules:
131+
importlib.import_module(m)
132+
133+
asyncio.run(_start_grpc_server("Connector", port, prometheus_port, worker, timeout))
134+
135+
136+
async def _start_grpc_server(name: str, port: int, prometheus_port: int, worker: int, timeout: int):
81137
try:
82-
from flytekit.extend.backend.agent_service import AgentMetadataService, AsyncAgentService, SyncAgentService
83-
from flytekit.extras.webhook import WebhookAgent # noqa: F401 Webhook Agent Registration
138+
from flytekit.extend.backend.connector_service import (
139+
AsyncConnectorService,
140+
ConnectorMetadataService,
141+
SyncConnectorService,
142+
)
143+
from flytekit.extras.webhook import WebhookConnector # noqa: F401 Webhook Connector Registration
84144
except ImportError as e:
85145
raise ImportError(
86-
"Flyte agent dependencies are not installed. Please install it using `pip install flytekit[agent]`"
146+
f"Flyte connector dependencies are not installed. Please install it using `pip install flytekit[{name.lower()}]`"
87147
) from e
88148

89-
click.secho("🚀 Starting the agent service...")
149+
click.secho(f"🚀 Starting the {name.lower()} service...")
90150
_start_http_server(prometheus_port)
91151

92-
print_agents_metadata()
152+
print_metadata(name)
93153

94154
server = grpc.aio.server(futures.ThreadPoolExecutor(max_workers=worker))
95155

96-
add_AsyncAgentServiceServicer_to_server(AsyncAgentService(), server)
97-
add_SyncAgentServiceServicer_to_server(SyncAgentService(), server)
98-
add_AgentMetadataServiceServicer_to_server(AgentMetadataService(), server)
156+
add_AsyncAgentServiceServicer_to_server(AsyncConnectorService(), server)
157+
add_SyncAgentServiceServicer_to_server(SyncConnectorService(), server)
158+
add_AgentMetadataServiceServicer_to_server(ConnectorMetadataService(), server)
99159
_start_health_check_server(server, worker)
100160

101161
server.add_insecure_port(f"[::]:{port}")
@@ -132,21 +192,21 @@ def _start_health_check_server(server: grpc.Server, worker: int):
132192
click.secho(f"Failed to start the health check servicer with error {e}", fg="red")
133193

134194

135-
def print_agents_metadata():
136-
from flytekit.extend.backend.base_agent import AgentRegistry
195+
def print_metadata(name: str):
196+
from flytekit.extend.backend.base_connector import ConnectorRegistry
137197

138-
agents = AgentRegistry.list_agents()
198+
connectors = ConnectorRegistry.list_connectors()
139199

140-
table = Table(title="Agent Metadata")
141-
table.add_column("Agent Name", style="cyan", no_wrap=True)
200+
table = Table(title=f"{name} Metadata")
201+
table.add_column(f"{name} Name", style="cyan", no_wrap=True)
142202
table.add_column("Support Task Types", style="cyan")
143203
table.add_column("Is Sync", style="green")
144204

145-
for a in agents:
205+
for connector in connectors:
146206
categories = ""
147-
for c in a.supported_task_categories:
148-
categories += f"{c.name} (v{c.version}) "
149-
table.add_row(a.name, categories, str(a.is_sync))
207+
for category in connector.supported_task_categories:
208+
categories += f"{category.name} ({category.version}) "
209+
table.add_row(connector.name, categories, str(connector.is_sync))
150210

151211
console = Console()
152212
console.print(table)

Diff for: ‎flytekit/exceptions/system.py

+4
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ class FlyteAgentNotFound(FlyteSystemException, AssertionError):
5151
_ERROR_CODE = "SYSTEM:AgentNotFound"
5252

5353

54+
class FlyteConnectorNotFound(FlyteSystemException, AssertionError):
55+
_ERROR_CODE = "SYSTEM:ConnectorNotFound"
56+
57+
5458
class FlyteDownloadDataException(FlyteSystemException):
5559
_ERROR_CODE = "SYSTEM:DownloadDataError"
5660

0 commit comments

Comments
 (0)