Skip to content

Commit ebe6e40

Browse files
oathdruidoathdruid
committed
re-oganize pipeline
1 parent 9621841 commit ebe6e40

File tree

12 files changed

+8761
-109
lines changed

12 files changed

+8761
-109
lines changed

‎.github/workflows/ci.yml

+47-108
Original file line numberDiff line numberDiff line change
@@ -2,139 +2,76 @@ name: CI
22

33
on:
44
push:
5-
branches: ["main"]
5+
branches: [main]
66
pull_request:
7-
branches: ["main"]
7+
branches: [main]
88

99
jobs:
10-
gcc12-basic-asan:
10+
basic:
11+
strategy:
12+
matrix:
13+
compiler: [{name: gcc, flag: --action_env=CC=gcc-12}, {name: clang, flag: --action_env=CC=clang-14}]
14+
std: [{name: c++17}, {name: c++14, flag: --cxxopt=-std=c++14 --cxxopt=-faligned-new}]
15+
stdlib: [{name: stdlibc++}, {name: libc++, flag: --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++}]
16+
feature: [{name: asan, flag: --features=asan}, {name: tsan, flag: --features=tsan}]
17+
exclude:
18+
- compiler: {name: gcc}
19+
stdlib: {name: libc++}
20+
- std: {name: c++14}
21+
feature: {name: tsan}
1122
runs-on: ubuntu-latest
23+
name: basic-${{matrix.compiler.name}}-${{matrix.std.name}}-${{matrix.stdlib.name}}-${{matrix.feature.name}}
1224
steps:
1325
- uses: actions/checkout@v4
1426
- uses: actions/cache@v4
1527
with:
1628
path: bazel-disk
17-
key: bazel-disk-gcc12-basic-asan-${{github.sha}}
18-
restore-keys: bazel-disk-gcc12-basic-asan-
19-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=gcc-12 --features=asan test/...
29+
key: bazel-disk-basic-${{matrix.compiler.name}}-${{matrix.std.name}}-${{matrix.stdlib.name}}-${{matrix.feature.name}}-${{github.sha}}
30+
restore-keys: bazel-disk-basic-${{matrix.compiler.name}}-${{matrix.std.name}}-${{matrix.stdlib.name}}-${{matrix.feature.name}}-
31+
save-always: true
32+
- run: bazel test --disk_cache=bazel-disk --verbose_failures ${{matrix.compiler.flag}} ${{matrix.std.flag}} ${{matrix.stdlib.flag}} ${{matrix.feature.flag}} test/...
2033

21-
gcc12-basic-tsan:
34+
arenastring:
35+
strategy:
36+
matrix:
37+
compiler: [{name: gcc, flag: --action_env=CC=gcc-12}, {name: clang, flag: --action_env=CC=clang-14}]
38+
stdlib: [{name: stdlibc++}, {name: libc++, flag: --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++}]
39+
mutable: [{name: default}, {name: half, flag: --config=arenastring}, {name: full, flag: --config=mutable-donated-string}]
40+
exclude:
41+
- compiler: {name: gcc}
42+
stdlib: {name: libc++}
43+
- compiler: {name: clang}
44+
mutable: {name: half}
45+
- compiler: {name: clang}
46+
mutable: {name: full}
2247
runs-on: ubuntu-latest
48+
name: arenastring-${{matrix.compiler.name}}-${{matrix.stdlib.name}}-${{matrix.mutable.name}}
2349
steps:
2450
- uses: actions/checkout@v4
2551
- uses: actions/cache@v4
2652
with:
2753
path: bazel-disk
28-
key: bazel-disk-gcc12-basic-tsan-${{github.sha}}
29-
restore-keys: bazel-disk-gcc12-basic-tsan-
30-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=gcc-12 --features=tsan test/...
54+
key: bazel-disk-arenastring-${{matrix.compiler.name}}-${{matrix.stdlib.name}}-${{matrix.mutable.name}}-${{github.sha}}
55+
restore-keys: bazel-disk-arenastring-${{matrix.compiler.name}}-${{matrix.stdlib.name}}-${{matrix.mutable.name}}-
56+
save-always: true
57+
- run: sed -i "/single_version_override.*protobuf/s/version = '[^']*'/version = '27.3.arenastring'/" MODULE.bazel
58+
- run: bazel test --disk_cache=bazel-disk --verbose_failures --features=asan ${{matrix.compiler.flag}} ${{matrix.stdlib.flag}} ${{matrix.mutable.flag}} test/...
3159

32-
gcc12-arenastring-asan:
60+
aarch64:
3361
runs-on: ubuntu-latest
3462
steps:
3563
- uses: actions/checkout@v4
3664
- uses: actions/cache@v4
3765
with:
3866
path: bazel-disk
39-
key: bazel-disk-gcc12-arenastring-${{github.sha}}
40-
restore-keys: bazel-disk-gcc12-arenastring-
41-
- run: sed -i "/single_version_override.*protobuf/s/version = '[^']*'/version = '25.3.arenastring'/" MODULE.bazel
42-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=gcc-12 --config=arenastring --features=asan test/...
43-
44-
gcc12-mutable-donated-string-asan:
45-
runs-on: ubuntu-latest
46-
steps:
47-
- uses: actions/checkout@v4
48-
- uses: actions/cache@v4
49-
with:
50-
path: bazel-disk
51-
key: bazel-disk-gcc12-mutable-donated-string-${{github.sha}}
52-
restore-keys: bazel-disk-gcc12-mutable-donated-string-
53-
- run: sed -i "/single_version_override.*protobuf/s/version = '[^']*'/version = '25.3.arenastring'/" MODULE.bazel
54-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=gcc-12 --config=mutable-donated-string --features=asan test/...
55-
56-
gcc12-cxx14-asan:
57-
runs-on: ubuntu-latest
58-
steps:
59-
- uses: actions/checkout@v4
60-
- uses: actions/cache@v4
61-
with:
62-
path: bazel-disk
63-
key: bazel-disk-gcc12-cxx14-asan-${{github.sha}}
64-
restore-keys: bazel-disk-gcc12-cxx14-asan-
65-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=gcc-12 --cxxopt=-std=c++14 --cxxopt=-faligned-new --features=asan test/...
66-
67-
clang14-basic-asan:
68-
runs-on: ubuntu-latest
69-
steps:
70-
- uses: actions/checkout@v4
71-
- uses: actions/cache@v4
72-
with:
73-
path: bazel-disk
74-
key: bazel-disk-clang14-basic-asan-${{github.sha}}
75-
restore-keys: bazel-disk-clang14-basic-asan-
76-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=clang-14 --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ --features=asan test/...
77-
78-
clang14-basic-tsan:
79-
runs-on: ubuntu-latest
80-
steps:
81-
- uses: actions/checkout@v4
82-
- uses: actions/cache@v4
83-
with:
84-
path: bazel-disk
85-
key: bazel-disk-clang14-basic-tsan-${{github.sha}}
86-
restore-keys: bazel-disk-clang14-basic-tsan-
87-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=clang-14 --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ --features=tsan test/...
88-
89-
clang14-arenastring-asan:
90-
runs-on: ubuntu-latest
91-
steps:
92-
- uses: actions/checkout@v4
93-
- uses: actions/cache@v4
94-
with:
95-
path: bazel-disk
96-
key: bazel-disk-clang14-arenastring-${{github.sha}}
97-
restore-keys: bazel-disk-clang14-arenastring-
98-
- run: sed -i "/single_version_override.*protobuf/s/version = '[^']*'/version = '25.3.arenastring'/" MODULE.bazel
99-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=clang-14 --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ --config=arenastring --features=asan test/...
100-
101-
clang14-mutable-donated-string-asan:
102-
runs-on: ubuntu-latest
103-
steps:
104-
- uses: actions/checkout@v4
105-
- uses: actions/cache@v4
106-
with:
107-
path: bazel-disk
108-
key: bazel-disk-clang14-mutable-donated-string-${{github.sha}}
109-
restore-keys: bazel-disk-clang14-mutable-donated-string-
110-
- run: sed -i "/single_version_override.*protobuf/s/version = '[^']*'/version = '25.3.arenastring'/" MODULE.bazel
111-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=clang-14 --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ --config=mutable-donated-string --features=asan test/...
112-
113-
clang14-cxx14-asan:
114-
runs-on: ubuntu-latest
115-
steps:
116-
- uses: actions/checkout@v4
117-
- uses: actions/cache@v4
118-
with:
119-
path: bazel-disk
120-
key: bazel-disk-clang14-cxx14-asan-${{github.sha}}
121-
restore-keys: bazel-disk-clang14-cxx14-asan-
122-
- run: bazel test --disk_cache=bazel-disk --action_env=CC=clang-14 --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ --cxxopt=-std=c++14 --cxxopt=-faligned-new --features=asan test/...
123-
124-
gcc12-aarch64-asan:
125-
runs-on: ubuntu-latest
126-
steps:
127-
- uses: actions/checkout@v4
128-
- uses: actions/cache@v4
129-
with:
130-
path: bazel-disk
131-
key: bazel-disk-gcc12-aarch64-asan-${{github.sha}}
132-
restore-keys: bazel-disk-gcc12-aarch64-asan-
67+
key: bazel-disk-aarch64-${{github.sha}}
68+
restore-keys: bazel-disk-aarch64-
69+
save-always: true
13370
- run: sudo apt update
13471
- run: sudo apt install g++-12-aarch64-linux-gnu
13572
- run: sudo apt install qemu-user
13673
- run: sudo ln -s /usr/aarch64-linux-gnu/lib/ld-linux-aarch64.so.1 /lib/
137-
- run: bazel test --disk_cache=bazel-disk --platforms='@cross_config_toolchain//:cross' --action_env=CROSS_CC=/usr/bin/aarch64-linux-gnu-gcc-12 --features=-default_link_flags --features=asan --linkopt=-fuse-ld=gold --test_env=LD_LIBRARY_PATH=/usr/aarch64-linux-gnu/lib --test_env=ASAN_OPTIONS=detect_odr_violation=1:detect_leaks=0 test/...
74+
- run: bazel test --disk_cache=bazel-disk --verbose_failures --platforms='@cross_config_toolchain//:cross' --action_env=CROSS_CC=/usr/bin/aarch64-linux-gnu-gcc-12 --features=-default_link_flags --test_env=LD_LIBRARY_PATH=/usr/aarch64-linux-gnu/lib test/...
13875

13976
workspace:
14077
runs-on: ubuntu-latest
@@ -145,7 +82,8 @@ jobs:
14582
path: bazel-disk
14683
key: bazel-disk-workspace-${{github.sha}}
14784
restore-keys: bazel-disk-workspace-
148-
- run: bazel test --enable_bzlmod=false --disk_cache=bazel-disk --action_env=CC=gcc-12 --features=asan test/...
85+
save-always: true
86+
- run: bazel test --disk_cache=bazel-disk --enable_bzlmod=false test/...
14987

15088
coverage:
15189
runs-on: ubuntu-latest
@@ -156,7 +94,8 @@ jobs:
15694
path: bazel-disk
15795
key: bazel-disk-coverage-${{github.sha}}
15896
restore-keys: bazel-disk-coverage-
159-
- run: bazel coverage --disk_cache=bazel-disk --combined_report=lcov --instrumentation_filter='src/babylon,-src/babylon/reusable/patch' test/...
97+
save-always: true
98+
- run: bazel coverage --disk_cache=bazel-disk --verbose_failures --combined_report=lcov --instrumentation_filter='src/babylon,-src/babylon/reusable/patch' test/...
16099
- run: .github/workflows/filter_lcov.sh
161100
- uses: coverallsapp/github-action@v2
162101
with:

‎MODULE.bazel

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ bazel_dep(name = 'boost.spirit', version = '1.83.0.bzl.1')
2525

2626
# --registry=https://bcr.bazel.build
2727
single_version_override(module_name = 'abseil-cpp', version = '20240116.2')
28-
single_version_override(module_name = 'protobuf', version = '27.2')
28+
single_version_override(module_name = 'protobuf', version = '27.3')
2929
bazel_dep(name = 'googletest', version = '1.14.0', repo_name = 'com_google_googletest', dev_dependency = True)
3030
bazel_dep(name = 'platforms', version = '0.0.10', dev_dependency = True)
3131
bazel_dep(name = 'rules_cc', version = '0.0.9', dev_dependency = True)

‎WORKSPACE.bzlmod

Whitespace-only changes.

‎docs/concurrent/execution_queue.md

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# execution_queue
2+
3+
## 原理
4+
5+
包装[ConcurrentBoundedQueue](bounded_queue.md)，实现按需激活MPSC消费者的模式
6+
- 每次生产数据时对待处理计数器原子自增，如果自增前为0，边沿触发，启动消费者线程
7+
- 消费者线程持续消费数据，当无数据可消费时，将计数器值和0交换
8+
- 如果交换得到的值和最近一次消费前记录的值无变化，退出消费者线程
9+
- 否则，进入下一轮消费循环
10+
11+
主要用于支持大量低活队列的情况，节省不活跃的监听消费线程，功能和[bthread::ExecutionQueue](https://github.com/apache/brpc/blob/master/docs/cn/execution_queue.md)类似，但是
12+
- 消费者通过[Executor](../executor.md)接口提交，支持使用自定义线程/协程机制
13+
- 保持生产者wait-free提交的同时，不会在竞争时引起消费者阻塞
14+
15+
![](images/bthread_execution_queue.md)
16+
17+
## 用法示例
18+
19+
```c++
20+
#include "babylon/concurrent/execution_queue.h"
21+
22+
using ::babylon::ConcurrentExecutionQueue;
23+
24+
// 显式定义一个队列
25+
using Queue = ConcurrentExecutionQueue<T>;
26+
Queue queue;
27+
28+
// 设置队列容量到N
29+
// 消费者采用some_executor执行
30+
// 注册lambda消费函数
31+
queue.initialize(N, some_executor, [] (Queue::Iterator iter, Queue::Iterator end) {
32+
// 消费区间的数据
33+
while (iter != end) {
34+
T& item = *iter;
35+
do_sth_with(item);
36+
++iter;
37+
}
38+
});
39+
40+
// 生产一个数据，并按需启动后台消费
41+
queue.execute("10086");
42+
...
43+
44+
// 等待当前发布数据全部消费完成
45+
// 注意不包含stop语义，可以反复execute & join
46+
queue.join();
47+
```

‎docs/concurrent/images/bthread_execution_queue.png

19.4 KB
Loading

‎docs/concurrent/index.md

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
- [bounded_queue](bounded_queue.md)
66
- [counter](counter.md)
7+
- [execution_queue](execution_queue.md)
78
- [id_allocator](id_allocator.md)
89
- [object_pool](object_pool.md)
910
- [thread_local](thread_local.md)

‎registry/modules/protobuf/27.3.arenastring/MODULE.bazel

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel
2+
# https://github.com/protocolbuffers/protobuf/issues/14313
3+
module(
4+
name = "protobuf",
5+
version = "27.3.arenastring", # Automatically updated on release
6+
compatibility_level = 1,
7+
repo_name = "com_google_protobuf",
8+
)
9+
10+
# LOWER BOUND dependency versions.
11+
# Bzlmod follows MVS:
12+
# https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution
13+
# Thus the highest version in their module graph is resolved.
14+
bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl")
15+
bazel_dep(name = "bazel_skylib", version = "1.4.1")
16+
bazel_dep(name = "jsoncpp", version = "1.9.5")
17+
bazel_dep(name = "rules_cc", version = "0.0.9")
18+
bazel_dep(name = "rules_java", version = "5.3.5")
19+
bazel_dep(name = "rules_jvm_external", version = "5.1")
20+
bazel_dep(name = "rules_pkg", version = "0.7.0")
21+
bazel_dep(name = "rules_python", version = "0.10.2")
22+
bazel_dep(name = "platforms", version = "0.0.8")
23+
bazel_dep(name = "zlib", version = "1.2.11")
24+
25+
# TODO: remove after toolchain types are moved to protobuf
26+
bazel_dep(name = "rules_proto", version = "4.0.0")
27+
28+
bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True)
29+
bazel_dep(name = "rules_ruby", dev_dependency = True)
30+
archive_override(
31+
module_name = "rules_ruby",
32+
urls = ["https://github.com/protocolbuffers/rules_ruby/archive/b7f3e9756f3c45527be27bc38840d5a1ba690436.tar.gz"],
33+
strip_prefix = "rules_ruby-b7f3e9756f3c45527be27bc38840d5a1ba690436",
34+
integrity = "sha256-RNo21X/p9slOdF912FLprLnxvZHMXBTxaUDGFmm2bx8=",
35+
patches = ["third_party/rules_ruby.patch"],
36+
patch_strip = 1,
37+
)

0 commit comments

Comments
 (0)