diff --git a/Makefile b/Makefile index 423b2405673b..17884025abe5 100644 --- a/Makefile +++ b/Makefile @@ -281,7 +281,7 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery $(ENV_INSTALL) apisix/discovery/*.lua $(ENV_INST_LUADIR)/apisix/discovery/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul,consul_kv,dns,eureka,nacos,kubernetes,tars} + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/discovery/{consul,consul_kv,dns,eureka,nacos,kubernetes,tars,zookeeper} $(ENV_INSTALL) apisix/discovery/consul/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul $(ENV_INSTALL) apisix/discovery/consul_kv/*.lua $(ENV_INST_LUADIR)/apisix/discovery/consul_kv $(ENV_INSTALL) apisix/discovery/dns/*.lua $(ENV_INST_LUADIR)/apisix/discovery/dns @@ -289,6 +289,7 @@ install: runtime $(ENV_INSTALL) apisix/discovery/kubernetes/*.lua $(ENV_INST_LUADIR)/apisix/discovery/kubernetes $(ENV_INSTALL) apisix/discovery/nacos/*.lua $(ENV_INST_LUADIR)/apisix/discovery/nacos $(ENV_INSTALL) apisix/discovery/tars/*.lua $(ENV_INST_LUADIR)/apisix/discovery/tars + $(ENV_INSTALL) apisix/discovery/zookeeper/*.lua $(ENV_INST_LUADIR)/apisix/discovery/zookeeper $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http $(ENV_INSTALL) apisix/http/*.lua $(ENV_INST_LUADIR)/apisix/http/ diff --git a/apisix/discovery/zookeeper/init.lua b/apisix/discovery/zookeeper/init.lua new file mode 100644 index 000000000000..0d524e29f7de --- /dev/null +++ b/apisix/discovery/zookeeper/init.lua @@ -0,0 +1,209 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local utils = require("apisix.discovery.zookeeper.utils") +local schema = require("apisix.discovery.zookeeper.schema") +local table = require("apisix.core.table") +local ngx = ngx +local ipairs = ipairs +local log = core.log + +local _M = { + version = 0.1, + priority = 1000, + name = "zookeeper", + schema = schema.schema, +} + +-- Global Configuration +local local_conf +-- Service Instance Cache(service_name -> {nodes, expire_time}) +local instance_cache = core.lrucache.new({ + ttl = 3600, + count = 1024 +}) + +-- Timer Identifier +local fetch_timer + +-- The instance list of a single service from ZooKeeper +local function fetch_service_instances(conf, service_name) + -- 1. Init connect + local client, err = utils.new_zk_client(conf) + if not client then + return nil, err + end + + -- 2. TODO: Create path + local service_path = conf.root_path .. "/" .. service_name + local ok, err = utils.create_zk_path(client, service_path) + if not ok then + utils.close_zk_client(client) + return nil, err + end + + -- 3. All instance nodes under a service + local children, err = client:get_children(service_path) + if not children then + utils.close_zk_client(client) + if err == "not exists" then + log.warn("service path not exists: ", service_path) + return {} + end + log.error("get zk children failed: ", err) + return nil, err + end + + -- 4. Parse the data of each instance node one by one + local instances = {} + for _, child in ipairs(children) do + local instance_path = service_path .. "/" .. child + local data, stat, err = client:get(instance_path) + if not data then + log.error("get instance data failed: ", instance_path, " stat:", stat, " err: ", err) + goto continue + end + + -- Parse instance data + local instance = utils.parse_instance_data(data) + if instance then + table.insert(instances, instance) + end + + ::continue:: + end + + -- 5. Close connects + utils.close_zk_client(client) + + log.debug("fetch service instances: ", service_name, " count: ", #instances) + return instances +end + +-- Scheduled fetch of all service instances (full cache update)) +local function fetch_all_services() + if not local_conf then + log.warn("zookeeper discovery config not loaded") + return + end + + -- 1. Init Zookeeper client + local client, err = utils.new_zk_client(local_conf) + if not client then + log.error("init zk client failed: ", err) + return + end + + -- 2. All instance nodes under a service + local services, err = client:get_children(local_conf.root_path) + if not services then + utils.close_zk_client(client) + log.error("get zk root children failed: ", err) + return + end + + -- 3. Fetch the instances of each service and update the cache + local now = ngx.time() + for _, service in ipairs(services) do + local instances, err = fetch_service_instances(local_conf, service) + if instances then + instance_cache:set(service, nil, { + nodes = instances, + expire_time = now + local_conf.cache_ttl + }) + else + log.error("fetch service instances failed: ", service, " err: ", err) + end + end + + -- 4. Close connects + utils.close_zk_client(client) +end + +function _M.nodes(service_name) + if not service_name or service_name == "" then + log.error("service name is empty") + return nil + end + + -- 1. Get from cache + local cache = instance_cache:get(service_name) + local now = ngx.time() + + -- 2. If the cache is missed or expired, actively pull (the data)) + if not cache or cache.expire_time < now then + log.debug("cache miss or expired, fetch from zk: ", service_name) + local instances, err = fetch_service_instances(local_conf, service_name) + if not instances then + log.error("fetch instances failed: ", service_name, " err: ", err) + -- Fallback: Return the old cache (if available)) + if cache then + return cache.nodes + end + return nil + end + + -- Update the cache + cache = { + nodes = instances, + expire_time = now + local_conf.cache_ttl + } + instance_cache:set(service_name, nil, cache) + end + + return cache.nodes +end + +function _M.check_schema(conf) + return schema.check(conf) +end + +function _M.init_worker() + -- Load configuration + local core_config = core.config.local_conf + local_conf = core_config.discovery and core_config.discovery.zookeeper or {} + local ok, err = schema.check(local_conf) + if not ok then + log.error("invalid zookeeper discovery config: ", err) + return + end + + -- The default values + local_conf.connect_string = local_conf.connect_string or "127.0.0.1:2181" + local_conf.fetch_interval = local_conf.fetch_interval or 10 + local_conf.cache_ttl = local_conf.cache_ttl or 30 + + -- Start the timer + if not fetch_timer then + fetch_timer = ngx.timer.every(local_conf.fetch_interval, fetch_all_services) + log.info("zk discovery fetch timer started, interval: ", local_conf.fetch_interval, "s") + end + + -- Manually execute a full pull immediately + ngx.timer.at(0, fetch_all_services) +end + +function _M.destroy() + if fetch_timer then + fetch_timer = nil + end + instance_cache:flush_all() + log.info("zookeeper discovery destroyed") +end + +return _M diff --git a/apisix/discovery/zookeeper/schema.lua b/apisix/discovery/zookeeper/schema.lua new file mode 100644 index 000000000000..43cf74badc2e --- /dev/null +++ b/apisix/discovery/zookeeper/schema.lua @@ -0,0 +1,91 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") + +local schema = { + type = "object", + properties = { + -- ZooKeeper Cluster Addresses (separated by commas for multiple addresses) + connect_string = { + type = "string", + default = "127.0.0.1:2181" + }, + -- ZooKeeper Session Timeout (milliseconds) + session_timeout = { + type = "integer", + minimum = 1000, + default = 30000 + }, + -- ZooKeeper Connect Timeout (milliseconds) + connect_timeout = { + type = "integer", + minimum = 1000, + default = 5000 + }, + -- Service Discovery Root Path + root_path = { + type = "string", + default = "/apisix/discovery/zk" + }, + -- Instance Fetch Interval (seconds) + fetch_interval = { + type = "integer", + minimum = 1, + default = 10 + }, + -- The default weight value for service instances that do not specify a weight in ZooKeeper. + -- It is used for load balancing (higher weight means more traffic). + -- Default value is 100, and the value range is 1-500. + weight = { + type = "integer", + minimum = 1, + default = 100 + }, + -- ZooKeeper Authentication Information (digest: username:password): + -- Digest authentication credentials for accessing ZooKeeper cluster. + -- Format requirement: "digest:{username}:{password}". + -- Leave empty to disable authentication (not recommended for production). + auth = { + type = "object", + properties = { + type = {type = "string", enum = {"digest"}, default = "digest"}, + creds = {type = "string"} -- digest: username:password + } + }, + -- Cache Expiration Time (seconds): + -- The time after which service instance cache becomes expired. + -- Default value is 60 seconds + cache_ttl = { + type = "integer", + minimum = 1, + default = 60 + } + }, + required = {}, + additionalProperties = false +} + +local _M = { + schema = schema +} + +function _M.check(conf) + return core.schema.check(schema, conf) +end + +return _M diff --git a/apisix/discovery/zookeeper/utils.lua b/apisix/discovery/zookeeper/utils.lua new file mode 100644 index 000000000000..aa36c71fa42e --- /dev/null +++ b/apisix/discovery/zookeeper/utils.lua @@ -0,0 +1,108 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local zk = require("resty.zookeeper") +local cjson = require("cjson.safe") +local ipairs = ipairs +local tonumber = tonumber + +local _M = {} + +-- Init ZK Clients +function _M.new_zk_client(conf) + local client, err = zk:new() + if not client then + core.log.error("failed to create zk client: ", err) + return nil, err + end + + client:set_timeout(conf.connect_timeout) + + local ok, err = client:connect(conf.connect_string, conf.session_timeout) + if not ok then + core.log.error("failed to connect to zk: ", err) + return nil, err + end + + if conf.auth and conf.auth.creds then + local ok, err = client:add_auth(conf.auth.type, conf.auth.creds) + if not ok then + core.log.warn("zk auth failed: ", err) + end + end + + return client +end + +-- Recursively Create ZooKeeper Nodes +function _M.create_zk_path(client, path) + local parts = core.utils.split(path, "/") + local current = "" + for _, part in ipairs(parts) do + if part ~= "" then + current = current .. "/" .. part + local exists, err = client:exists(current) + if err then + core.log.error("check zk path exists failed: ", err) + return false, err + end + if not exists then + local ok, err = client:create(current, "", "persistent", true) + if not ok and err ~= "node already exists" then + core.log.error("create zk path failed: ", current, " err: ", err) + return false, err + end + end + end + end + return true +end + +-- : Map ZK instance fields to APISIX +function _M.parse_instance_data(data) + local instance = cjson.decode(data) + if not instance then + core.log.error("invalid instance data: ", data) + return nil + end + + -- Validate Required Fields + if not instance.host or not instance.port then + core.log.error("instance missing host/port: ", cjson.encode(instance)) + return nil + end + + return { + host = instance.host, + port = tonumber(instance.port) or 80, + weight = tonumber(instance.weight) or 100, + metadata = instance.metadata + } +end + +-- Close ZK Clients +function _M.close_zk_client(client) + if client then + local ok, err = client:close() + if not ok then + core.log.error("close zk client failed: ", err) + end + end +end + +return _M diff --git a/docs/en/latest/discovery/zookeeper.md b/docs/en/latest/discovery/zookeeper.md new file mode 100644 index 000000000000..fca305912eda --- /dev/null +++ b/docs/en/latest/discovery/zookeeper.md @@ -0,0 +1,184 @@ +--- +title: Zookeeper +keywords: + - API Gateway + - Apache APISIX + - ZooKeeper + - Service Discovery +description: This documentation describes implementing service discovery through ZooKeeper on the API Gateway Apache APISIX. +--- + + + +## Service Discovery via ZooKeeper + +Apache APISIX supports integrating with ZooKeeper for service discovery. This allows APISIX to dynamically fetch service instance information from ZooKeeper and route requests accordingly. + +## Configuration for ZooKeeper + +To enable ZooKeeper service discovery, add the following configuration to `conf/config.yaml`: + +```yaml +discovery: + zookeeper: + connect_string: "127.0.0.1:2181,127.0.0.1:2182" # ZooKeeper Cluster Addresses (separated by commas for multiple addresses) + fetch_interval: 10 # Interval (in seconds) for fetching service data. Default: 10s + weight: 100 # Default weight for service instances. Default value is 100, and the value range is 1-500. + cache_ttl: 30 # The time after which service instance cache becomes expired. Default: 60s + connect_timeout: 2000 # Connect timeout (in ms). Default: 5000ms + session_timeout: 30000 # Session Timeout (in ms). Default: 30000ms + root_path: "/apisix/discovery/zk" # Root path for service registration in ZooKeeper, default: "/apisix/discovery/zk" + auth: # ZooKeeper Authentication Information. Format requirement: "digest:{username}:{password}". + type: "digest" + creds: "username:password" +``` + +And you can config it in short by default value: + +```yaml +discovery: + zookeeper: + connect_string: "127.0.0.1:2181" +``` + +### Upstream setting + +#### L7 (HTTP/HTTPS) + +Here is an example of routing requests with the URI `/zookeeper/*` to a service named `APISIX-ZOOKEEPER` registered in ZooKeeper: + +:::note +You can fetch the `admin_key` from `config.yaml` and save to an environment variable with the following command: + +```bash +$ admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g') +``` + +::: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/routes/1 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "uri": "/zookeeper/*", + "upstream": { + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper" + } +}' +``` + +The formatted response as below: + +```json +{ + "node": { + "key": "/apisix/routes/1", + "value": { + "id": "1", + "create_time": 1690000000, + "status": 1, + "update_time": 1690000000, + "upstream": { + "hash_on": "vars", + "pass_host": "pass", + "scheme": "http", + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper" + }, + "priority": 0, + "uri": "/zookeeper/*" + } + } +} +``` + +#### L4 (TCP/UDP) + +ZooKeeper service discovery also supports L4 proxy. Here's an example configuration for TCP: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "remote_addr": "127.0.0.1", + "upstream": { + "scheme": "tcp", + "discovery_type": "zookeeper", + "service_name": "APISIX-ZOOKEEPER-TCP", + "type": "roundrobin" + } +}' +``` + +### discovery_args + +| Name | Type | Required | Default | Valid | Description | +| ------------ | ------ | ----------- | ------- | ----- | ------------------------------------------------------------ | +| root_path | string | optional | "/apisix/discovery/zk" | | Custom root path for the service in ZooKeeper | + +#### Specify Root Path + +Example of routing to a service under a custom root path: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/routes/2 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "uri": "/zookeeper/custom/*", + "upstream": { + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper", + "discovery_args": { + "root_path": "/custom/services" + } + } +}' + +``` + +The formatted response as below: + +```json +{ + "node": { + "key": "/apisix/routes/2", + "value": { + "id": "2", + "create_time": 1615796097, + "status": 1, + "update_time": 1615799165, + "upstream": { + "hash_on": "vars", + "pass_host": "pass", + "scheme": "http", + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper", + "discovery_args": { + "root_path": "/custom/services" + } + }, + "priority": 0, + "uri": "/zookeeper/*" + } + } +} +``` diff --git a/docs/zh/latest/discovery/zookeeper.md b/docs/zh/latest/discovery/zookeeper.md new file mode 100644 index 000000000000..8e072d2d501b --- /dev/null +++ b/docs/zh/latest/discovery/zookeeper.md @@ -0,0 +1,184 @@ +--- +title: Zookeeper +keywords: + - API 网关 + - Apache APISIX + - ZooKeeper + - 服务发现 +description: 本文档介绍了如何在 API 网关 Apache APISIX 上通过 ZooKeeper 实现服务发现。 +--- + + + +## 通过 ZooKeeper 实现服务发现 + +Apache APISIX 支持与 ZooKeeper 集成以实现服务发现。这使得 APISIX 能够从 ZooKeeper 动态获取服务实例信息,并据此进行请求路由。 + +## ZooKeeper 配置 + +要启用 ZooKeeper 服务发现,请在 `conf/config.yaml` 中添加以下配置: + +```yaml +discovery: + zookeeper: + connect_string: "127.0.0.1:2181,127.0.0.1:2182" # ZooKeeper 集群地址(多个地址用逗号分隔) + fetch_interval: 10 # 获取服务数据的间隔时间(秒)。默认值:10s + weight: 100 # 服务实例的默认权重。默认值为 100,取值范围是 1-500。 + cache_ttl: 30 # 服务实例缓存过期时间。默认值:60s + connect_timeout: 2000 # 连接超时时间(毫秒)。默认值:5000ms + session_timeout: 30000 # 会话超时时间(毫秒)。默认值:30000ms + root_path: "/apisix/discovery/zk" # ZooKeeper 中服务注册的根路径,默认值:"/apisix/discovery/zk" + auth: # ZooKeeper 认证信息。格式要求:"digest:{username}:{password}"。 + type: "digest" + creds: "username:password" +``` + +您也可以使用默认值进行简化配置: + +```yaml +discovery: + zookeeper: + connect_string: "127.0.0.1:2181" +``` + +### 上游设置 + +#### L7(HTTP/HTTPS) + +以下示例将 URI 为 `/zookeeper/*` 的请求路由到在 ZooKeeper 中注册的名为 `APISIX-ZOOKEEPER` 的服务: + +:::note + +您可以这样从 `config.yaml` 中获取 `admin_key` 并存入环境变量: + +```bash +$ admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g') +``` + +::: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/routes/1 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "uri": "/zookeeper/*", + "upstream": { + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper" + } +}' +``` + +格式化后的响应如下: + +```json +{ + "node": { + "key": "/apisix/routes/1", + "value": { + "id": "1", + "create_time": 1690000000, + "status": 1, + "update_time": 1690000000, + "upstream": { + "hash_on": "vars", + "pass_host": "pass", + "scheme": "http", + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper" + }, + "priority": 0, + "uri": "/zookeeper/*" + } + } +} +``` + +#### 四层(TCP/UDP) + +ZooKeeper 服务发现也支持 L4 代理。以下是 TCP 的配置示例: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/stream_routes/1 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "remote_addr": "127.0.0.1", + "upstream": { + "scheme": "tcp", + "discovery_type": "zookeeper", + "service_name": "APISIX-ZOOKEEPER-TCP", + "type": "roundrobin" + } +}' +``` + +### 参数 + +| 名字 | 类型 | 可选项 | 默认值 | 有效值 | 说明 | +| ------------ | ------ | ----------- | ------- | ----- | ------------------------------------------------------------ | +| root_path | string | 可选 | "/apisix/discovery/zk" | | ZooKeeper 中服务的自定义根路径 | + +#### 指定根路径 + +路由到自定义根路径下服务的示例: + +```shell +$ curl http://127.0.0.1:9180/apisix/admin/routes/2 -H "X-API-KEY: $admin_key" -X PUT -i -d ' +{ + "uri": "/zookeeper/custom/*", + "upstream": { + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper", + "discovery_args": { + "root_path": "/custom/services" + } + } +}' +``` + +格式化后的响应如下: + +```json +{ + "node": { + "key": "/apisix/routes/2", + "value": { + "id": "2", + "create_time": 1615796097, + "status": 1, + "update_time": 1615799165, + "upstream": { + "hash_on": "vars", + "pass_host": "pass", + "scheme": "http", + "service_name": "APISIX-ZOOKEEPER", + "type": "roundrobin", + "discovery_type": "zookeeper", + "discovery_args": { + "root_path": "/custom/services" + } + }, + "priority": 0, + "uri": "/zookeeper/*" + } + } +} +```