From 9b0b94aa927a44fc522d1dd1c506e08353043c3d Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Tue, 13 Feb 2024 15:51:39 -0800 Subject: [PATCH 01/27] MINOR: Revert global checkstyle metrics configuration to 2018 Signed-off-by: Greg Harris --- checkstyle/checkstyle.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index aff659638928b..04ec2ef9a3b9a 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -113,7 +113,7 @@ - + @@ -131,7 +131,7 @@ - + From 5232ff92c0313af5fc2ec5aa3e9a8989dc9861fa Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 16:23:33 -0800 Subject: [PATCH 02/27] Add suppressions for classes with MethodLength warnings Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 7486ef9a80d3e..5e1fea53721d5 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -315,6 +315,8 @@ files="(QuorumController|QuorumControllerTest|ReplicationControlManager|ReplicationControlManagerTest|ClusterControlManagerTest|KRaftMigrationDriverTest).java"/> + + Date: Thu, 15 Feb 2024 18:23:19 -0800 Subject: [PATCH 03/27] Remove duplicate suppressions --- checkstyle/suppressions.xml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 5e1fea53721d5..55d6b56f160db 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -40,7 +40,6 @@ files="core[\\/]src[\\/](generated|generated-test)[\\/].+.java$"/> - @@ -58,15 +57,13 @@ files="SaslServerAuthenticator.java"/> - + files="(KerberosLogin|ConnectMetricsRegistry|KafkaConsumer|AbstractStickyAssignor|AbstractRequest|AbstractResponse).java"/> @@ -95,10 +92,10 @@ files="(AbstractFetch|ConsumerCoordinator|CommitRequestManager|FetchCollector|OffsetFetcherUtils|KafkaProducer|Sender|ConfigDef|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor|DefaultSslEngineFactory|Authorizer|RecordAccumulator|MemoryRecords|FetchSessionHandler|MockAdminClient).java"/> + files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaRaftClientTest).java"/> + files="(ConsumerCoordinator|BufferPool|MetricName|Node|ConfigDef|RecordBatch|SslFactory|SslTransportLayer|MetadataResponse|KerberosLogin|Selector|Sender|Serdes|TokenInformation|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|AbstractStickyAssignor|KafkaRaftClient|Authorizer|FetchSessionHandler|RecordAccumulator).java"/> @@ -139,8 +136,6 @@ - + files="(DistributedHerder|Worker)Test.java"/> @@ -256,9 +251,6 @@ - - @@ -307,8 +299,6 @@ - - @@ -354,7 +344,7 @@ + files="(LogValidator|RemoteIndexCache).java"/> Date: Thu, 15 Feb 2024 18:24:32 -0800 Subject: [PATCH 04/27] fixup: remove duplicate Murmur3 Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 55d6b56f160db..308bcc5207690 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -99,8 +99,6 @@ - @@ -130,9 +128,6 @@ - - From aff8a8c549497aead93278ffcd5701237c6c3fd5 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 16:59:54 -0800 Subject: [PATCH 05/27] Normalize check names Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 308bcc5207690..75b8079a3a3cd 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -44,7 +44,7 @@ files="(KafkaClusterTestKit).java"/> - + - @@ -330,7 +330,7 @@ files="(GroupMetadataManager|GroupMetadataManagerTest|GroupMetadataManagerTestContext|GroupCoordinatorService|GroupCoordinatorServiceTest).java"/> - From e1d799644da5b429cf5af2facb101041b53a07c7 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:05:49 -0800 Subject: [PATCH 06/27] Normalize some suppressions Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 75b8079a3a3cd..cf898b2036d98 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -104,7 +104,7 @@ files="clients[\\/]src[\\/](generated|generated-test)[\\/].+.java$"/> + files="(MessageTest|OffsetFetchRequest).java"/> + files="(RequestResponseTest|FetcherTest|FetchRequestManagerTest|KafkaAdminClientTest).java"/> @@ -134,14 +134,14 @@ + files="Worker(SinkTask|SourceTask|Coordinator).java"/> + files="(Worker|ConfigKeyInfo|DistributedHerder).java"/> + files="(RestServer|AbstractHerder|DistributedHerder|Worker).java"/> @@ -159,7 +159,7 @@ + files="(DistributedHerder|KafkaBasedLog|WorkerSourceTaskWithTopicCreation|WorkerSourceTask|Worker)Test.java"/> @@ -254,7 +254,7 @@ + files="(PageViewTypedDemo|PipeDemo|TemperatureDemo|WordCountDemo|WordCountProcessorDemo|WordCountTransformerDemo).java"/> Date: Thu, 15 Feb 2024 17:30:16 -0800 Subject: [PATCH 07/27] system exit Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index cf898b2036d98..9124572e562fd 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -32,8 +32,6 @@ files="(ApiMessageType|FieldSpec|MessageDataGenerator|KafkaConsumerTest).java"/> - Date: Thu, 15 Feb 2024 17:36:52 -0800 Subject: [PATCH 08/27] class data abstraction coupling Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 9124572e562fd..602c243ebe3cd 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -79,9 +79,9 @@ files="ClientUtils.java"/> + files="(ConsumerCoordinator|KafkaProducer|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|AsyncKafkaConsumer).java"/> + files="(SaslAuthenticatorTest|AgentTest|NetworkClientTest).java"/> @@ -106,7 +106,7 @@ + files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> @@ -139,7 +139,7 @@ files="LoggingResource.java" /> + files="(AbstractHerder|DistributedHerder|Worker).java"/> @@ -157,7 +157,7 @@ + files="(DistributedHerder|Worker)Test.java"/> @@ -255,8 +255,6 @@ files="(PageViewTypedDemo|PipeDemo|TemperatureDemo|WordCountDemo|WordCountProcessorDemo|WordCountTransformerDemo).java"/> - Date: Thu, 15 Feb 2024 17:37:10 -0800 Subject: [PATCH 09/27] boolean expression --- checkstyle/suppressions.xml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 602c243ebe3cd..0bfb55addd89b 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -24,8 +24,10 @@ - + + files="(ByteUtils|Topic|KafkaLZ4BlockOutputStream).java"/> @@ -141,9 +143,6 @@ - - - Date: Thu, 15 Feb 2024 17:44:45 -0800 Subject: [PATCH 10/27] static variable name Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 0bfb55addd89b..bcd1994f8f7bf 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -189,9 +189,6 @@ - - From cfb3b7e70bdee08c1c2fda8bd2f384720379fd41 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:45:57 -0800 Subject: [PATCH 11/27] class fan out complexity --- checkstyle/suppressions.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index bcd1994f8f7bf..75aa4edb1f42b 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -50,17 +50,11 @@ + files="(Sender|SenderTest|ConsumerCoordinator|KafkaProducer|Utils|TransactionManager|TransactionManagerTest|KafkaAdminClient|NetworkClient|Admin|KafkaRaftClient|RaftClientTestContext|AsyncKafkaConsumer|LegacyKafkaConsumer).java"/> - - - @@ -217,7 +211,7 @@ + files="(RecordCollectorTest|StreamsPartitionAssignorTest|StreamThreadTest|StreamTaskTest|TaskManagerTest|KafkaStreamsTest|EosIntegrationTest).java"/> @@ -317,7 +311,7 @@ + files="(GroupMetadataManager|GroupMetadataManagerTest|GroupMetadataManagerTestContext|GroupCoordinatorServiceTest).java"/> Date: Thu, 15 Feb 2024 17:35:03 -0800 Subject: [PATCH 12/27] parameter number --- checkstyle/suppressions.xml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 75aa4edb1f42b..ca36f18bcd662 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -62,7 +62,7 @@ + files="(AsyncKafkaConsumer|ConsumerCoordinator).java"/> + files="Worker(SinkTask|SourceTask).java"/> + files="(Worker|DistributedHerder).java"/> @@ -255,8 +255,6 @@ files="SignalLogger.java"/> - + files="RemoteLogManagerConfig.java"/> From 0bb30796987dbc6ac197be0a079db3dadb572e07 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:55:18 -0800 Subject: [PATCH 13/27] ncss Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index ca36f18bcd662..2bc901e4465e7 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -31,7 +31,7 @@ + files="(FieldSpec|MessageDataGenerator|KafkaConsumerTest).java"/> @@ -143,7 +143,7 @@ files="(JsonConverter|Values|ConnectHeaders).java"/> + files="(Values|ConnectMetricsRegistry).java"/> From 3ce6b5b55d81b45373d07124fff905b4f4e1f797 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:56:06 -0800 Subject: [PATCH 14/27] cyclomatic complexity --- checkstyle/suppressions.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 2bc901e4465e7..beea711db72d2 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -83,7 +83,7 @@ files="(ByteUtils|Topic|KafkaLZ4BlockOutputStream).java"/> + files="(ConsumerCoordinator|CommitRequestManager|FetchCollector|OffsetFetcherUtils|KafkaProducer|Sender|ConfigDef|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager|AbstractStickyAssignor|Authorizer|FetchSessionHandler|MockAdminClient).java"/> @@ -111,7 +111,7 @@ files="MockAdminClient.java"/> + files="RequestResponseTest.java"/> @@ -138,7 +138,7 @@ files="(AbstractHerder|DistributedHerder|Worker).java"/> + files="(FileStreamSourceTask|DistributedHerder).java"/> @@ -181,7 +181,7 @@ files="(KafkaStreams|KStreamImpl|KTableImpl).java"/> + files="(KafkaStreams|StreamsPartitionAssignor|StreamThread|TaskManager|SubscriptionWrapperSerde|AssignorConfiguration).java"/> @@ -246,7 +246,7 @@ + files="(ConsoleConsumer|DefaultMessageFormatter|StreamsResetter|ProducerPerformance).java"/> - From 43d57aeac6535d85050cfd14f6a0c50d89251d11 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:57:10 -0800 Subject: [PATCH 15/27] npath complexity --- checkstyle/suppressions.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index beea711db72d2..16b0d7ba71dcd 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -89,7 +89,7 @@ files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaRaftClientTest).java"/> + files="(ConsumerCoordinator|BufferPool|AbstractLegacyRecordBatch|SslTransportLayer|KerberosLogin|Selector|Sender|Serdes|Values|PluginUtils|MiniTrogdorCluster|TasksRequest|KafkaProducer|KafkaRaftClient|Authorizer|FetchSessionHandler).java"/> @@ -97,9 +97,6 @@ - - @@ -116,9 +113,6 @@ - - @@ -146,7 +140,7 @@ files="(Values|ConnectMetricsRegistry).java"/> + files="(DistributedHerder|AbstractHerder|JsonConverter|FileStreamSourceTask|WorkerSourceTask|TopicAdmin).java"/> + files="(ClientQuotasImage|KafkaEventQueue|ReplicationControlManager|KRaftMigrationDriver|ScramControlManager|ClusterControlManager|MetadataDelta|MetaPropertiesEnsemble).java"/> Date: Thu, 15 Feb 2024 17:57:33 -0800 Subject: [PATCH 16/27] method length --- checkstyle/suppressions.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 16b0d7ba71dcd..dd8864253a08e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -57,7 +57,7 @@ files="SaslServerAuthenticator.java"/> + files="(KerberosLogin|ConnectMetricsRegistry|AbstractRequest|AbstractResponse).java"/> @@ -120,7 +120,7 @@ + files="(DistributedHerder|DistributedConfig|Values|IncrementalCooperativeAssignor).java"/> + files="WorkerSinkTaskTest.java"/> @@ -208,7 +208,7 @@ files="(RecordCollectorTest|StreamsPartitionAssignorTest|StreamThreadTest|StreamTaskTest|TaskManagerTest|KafkaStreamsTest|EosIntegrationTest).java"/> + files="(EosIntegrationTest|EosV2UpgradeIntegrationTest|KStreamKStreamJoinTest|RocksDBWindowStoreTest).java"/> From c4768a9ad8d373d153301bdbf73fb11690529ce0 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:58:10 -0800 Subject: [PATCH 17/27] import control --- checkstyle/suppressions.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index dd8864253a08e..5010f8d75577e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -200,9 +200,6 @@ - - - From 38d1f5336299be2c937cfac01a5bdb6c6a5c482e Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 18:27:03 -0800 Subject: [PATCH 18/27] whitespace, local variable name, import control --- checkstyle/suppressions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 5010f8d75577e..c84d6768bb38c 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -219,7 +219,7 @@ - Date: Thu, 15 Feb 2024 18:27:59 -0800 Subject: [PATCH 19/27] npath complexity and cyclomatic complexity --- checkstyle/suppressions.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index c84d6768bb38c..3d3bf12246489 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -178,14 +178,11 @@ files="(KafkaStreams|StreamsPartitionAssignor|StreamThread|TaskManager|SubscriptionWrapperSerde|AssignorConfiguration).java"/> + files="(KafkaStreams|StreamsPartitionAssignor|StreamThread|TaskManager|KStreamImplJoin|TopologyConfig|KStreamSlidingWindowAggregate).java"/> - - From 3e82558faf11cfebf875229ae21d171f2659cc3e Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 17:58:41 -0800 Subject: [PATCH 20/27] npath complexity, import control, illegal import --- checkstyle/suppressions.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 3d3bf12246489..c4e8b48151005 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -236,11 +236,7 @@ - - + files="(DefaultMessageFormatter|ProducerPerformance|StreamsResetter|ReplicaVerificationTool).java"/> Date: Thu, 15 Feb 2024 17:59:02 -0800 Subject: [PATCH 21/27] cyclomatic complexity, method length, ncss --- checkstyle/suppressions.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index c4e8b48151005..dcba360bb2f48 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -248,11 +248,7 @@ - - + files="GlobComponent.java"/> Date: Thu, 15 Feb 2024 18:00:19 -0800 Subject: [PATCH 22/27] npath complexity and method length --- checkstyle/suppressions.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index dcba360bb2f48..1a5918cf52e71 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -284,10 +284,10 @@ - + + files="(GroupMetadataManager|GeneralUniformAssignmentBuilder|CoordinatorRuntime).java"/> Date: Thu, 15 Feb 2024 18:04:04 -0800 Subject: [PATCH 23/27] data abstraction coupling, npath complexity, method length --- checkstyle/suppressions.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 1a5918cf52e71..3a2799837b0ed 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -39,9 +39,8 @@ - - + + From 6a68eeaeeb5d34c223340b7a6060cd63870ef14a Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 19:20:34 -0800 Subject: [PATCH 24/27] Add suppressions for classes that were covered by other suppressions Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 3a2799837b0ed..3e8ddb83f673e 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -49,7 +49,7 @@ + files="(Sender|SenderTest|ConsumerCoordinator|KafkaProducer|Utils|TransactionManager|TransactionManagerTest|KafkaAdminClient|NetworkClient|Admin|ForwardingAdmin|KafkaRaftClient|RaftClientTestContext|AsyncKafkaConsumer|LegacyKafkaConsumer).java"/> + files="(ConsumerCoordinator|Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|AsyncKafkaConsumer|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> + files="(ConsumerCoordinator|KafkaConsumer|AsyncKafkaConsumer|RequestResponse|Fetcher|OffsetFetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> + files="(IntegrationTestUtils|MockAdminClient).java"/> @@ -112,6 +112,8 @@ + + @@ -121,7 +123,7 @@ + files="(WorkerSinkTask|WorkerSourceTask|AbstractWorkerSourceTask|ExactlyOnceWorkerSourceTask).java"/> + files="(WorkerSource|AbstractWorkerSource|ExactlyOnceWorkerSource|ErrorHandling)TaskTest.java"/> From e8787ab818e6650e000a3d106586c1b400cdbbe4 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Thu, 15 Feb 2024 19:29:50 -0800 Subject: [PATCH 25/27] Add checkstyle_audit.py that verifies that global checkstyle suppressions are up-to-date. --- checkstyle_audit.py | 155 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 checkstyle_audit.py diff --git a/checkstyle_audit.py b/checkstyle_audit.py new file mode 100755 index 0000000000000..81706e3e01f16 --- /dev/null +++ b/checkstyle_audit.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import xml.etree.ElementTree as ET +from rstr import Rstr +from random import SystemRandom +import glob + +""" +Utility for auditing the contents of checkstyle/suppressions.xml to find suppressions which are no longer necessary. + +1. Install rstr (and other dependencies if you don't have them) + +2. Edit checkstyle.xml to remove the following node and disable all suppressions + + + + +3. Run the following command to run checkstyle on all modules without any suppressions (should fail) +./gradlew checkstyleMain checkstyleTest --continue + +4. Run the script as many times as necessary, iterating and editing suppressions.xml as you work through the warnings +./checkstyle_audit.py + +5. Revert the change to checkstyle.xml as it's only for the script to function correctly +git checkout HEAD checkstyle/checkstyle.xml + +6. Run checkstyle on the repo to verify the suppressions changes +./gradlew checkstyleMain checkstyleTest +""" + + +# For catching typos, enforcing consistency of rules +_known_checks = { + "ImportControl", + "AvoidStarImport", + "ClassDataAbstractionCoupling", + "ClassFanOutComplexity", + "FinalLocalVariable", + "ParameterNumber", + "NPathComplexity", + "BooleanExpressionComplexity", + "CyclomaticComplexity", + "dontUseSystemExit", + "JavaNCSS", + "StaticVariableName", + "MemberName", + "ParameterName", + "LocalVariableName", + "MethodLength", + "WhitespaceAfter", + "WhitespaceAround", + "UnnecessaryParentheses", + "DefaultComesLast" +} + + +def _re_to_set(rs, regex): + subs = regex.replace(".java", "[.]java") + out = set() + for _ in range(200): + s = rs.xeger(subs) + if len(s) != 0: + out.add(s) + expected = regex.count("|") + 1 + if len(out) != expected: + return set() + return out + + +def main(): + suppressions = ET.parse("checkstyle/suppressions.xml") + rs = Rstr(SystemRandom()) + expected = dict() + too_general = set() + total_errors = 0 + for suppression in suppressions.getroot(): + if "checks" in suppression.attrib: + checks = _re_to_set(rs, suppression.attrib["checks"]) + elif "id" in suppression.attrib: + checks = _re_to_set(rs, suppression.attrib["id"]) + else: + continue + if "files" not in suppression.attrib: + continue + files = _re_to_set(rs, suppression.attrib["files"]) + for check in checks: + if check not in _known_checks: + print("Unknown check", check) + for file in files: + if "generated" in file: + continue + pair = (check, file) + if pair in expected: + first = expected.get(pair) + print(pair, "covered by two overlapping rules") + print(first.attrib) + print(suppression.attrib) + expected[pair] = suppression + + total = len(expected) + + reports = glob.glob("**/build/reports/checkstyle/*.xml", recursive=True) + for report in reports: + failures = ET.parse(report) + for file in failures.getroot(): + path = file.attrib["name"] + if "generated" in path: + continue + filename = path.split("/")[-1] + for error in file: + className = error.attrib["source"] + check = className.split(".")[-1].replace("Check", "") + pair = (check, filename) + if pair[0] == "FinalLocalVariable" and "org/apache/kafka/streams" not in path: + continue + if pair[0] == "ClassDataAbstractionCoupling" and "streams" in path and "test" in path: + continue + total_errors = total_errors + 1 + if pair in expected: + expected[pair] = None + else: + too_general.add(pair) + + print(len(too_general), "/", total_errors, "errors missing suppressions") + for pair in too_general: + covered = [(check, shorter_file) for check, shorter_file in expected if check == pair[0] and shorter_file in pair[1]] + if len(covered) == 0: + print(pair, "is missing suppression (?)") + else: + print(pair, "is covered by other suppression", covered) + unused = {k: expected[k] for k in expected if expected[k] is not None} + print(len(unused), "/", total, "suppressions missing errors") + for pair in unused: + print("unused", pair) + + +if __name__ == "__main__": + main() From d9585e1c1af69613685e68c2bac8ecf056937fe7 Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Tue, 5 Mar 2024 17:09:58 -0800 Subject: [PATCH 26/27] fixup: IQv2IntegrationTest MethodLength suppression Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 093b4fb1a676e..d7078fb845438 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -204,7 +204,7 @@ files="(RecordCollectorTest|StreamsPartitionAssignorTest|StreamThreadTest|StreamTaskTest|TaskManagerTest|KafkaStreamsTest|EosIntegrationTest).java"/> + files="(EosIntegrationTest|EosV2UpgradeIntegrationTest|KStreamKStreamJoinTest|RocksDBWindowStoreTest|IQv2IntegrationTest).java"/> From 7a323d215997ec48c53ed658015ae8e589e84bfb Mon Sep 17 00:00:00 2001 From: Greg Harris Date: Fri, 29 Mar 2024 16:17:28 -0700 Subject: [PATCH 27/27] Remove extra suppressions, fix streams/test-utils main code path check Signed-off-by: Greg Harris --- checkstyle/suppressions.xml | 11 ++++------- checkstyle_audit.py | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 0553595dd7f17..4791773ab6654 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -44,7 +44,6 @@ - + files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest|KafkaRaftClientTest).java"/> @@ -113,8 +112,6 @@ - - @@ -149,10 +146,10 @@ files="(DistributedHerder|Worker)Test.java"/> + files="(WorkerSink|AbstractWorkerSource|ExactlyOnceWorkerSource|WorkerSource|ErrorHandling)TaskTest.java"/> + files="(DistributedHerder|Worker)Test.java"/> @@ -237,7 +234,7 @@ + files="(DefaultMessageFormatter|StreamsResetter|ProducerPerformance).java"/>