Cherry-picks for 1.11.2 release

.github/workflows/codeql-csharp-analysis.yml

@@ -67,6 +67,7 @@ jobs:
           3.1.x
           5.0.x
           6.0.x
+          7.0.x
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/maven4.yml

@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: 'Maven 4'
+on:
+  workflow_dispatch:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    paths:
+    - .github/workflows/maven4.yml
+    - lang/java/**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  maven4:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Cache Local Maven Repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Cache Maven 4 Build Cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/build-cache
+          key: ${{ runner.os }}-maven-build-cache-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-build-cache
+
+      - name: Setup Java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: '11'
+
+      - name: Setup Maven 4
+        uses: stCarolas/[email protected]
+        with:
+          maven-version: 4.0.0-alpha-3
+
+      - name: Test
+        run: mvn clean package

.github/workflows/test-lang-csharp.yml

@@ -36,21 +36,22 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Add libzstd
         shell: bash
         run: sudo apt-get install -y libzstd-dev
 
       - name: Install .NET SDKs
-        uses: actions/setup-dotnet@v1
+        uses: actions/setup-dotnet@v3
         with:
           dotnet-version: |
             3.1.x
             5.0.x
             6.0.x
+            7.0.x
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.nuget/packages
           key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
@@ -63,38 +64,26 @@ jobs:
       - name: Test
         run: ./build.sh test
 
-      # Build and test against .NET 7
-      # .NET 7 is not released yet, however this is a good way to test if the project is ready for the release
-      # Once .NET 7 is officially released, this can be removed and 7.0.x can be used instead above
-      - name: Install .NET SDK 7.0 (pre-release)
-        uses: actions/setup-dotnet@v1
-        with:
-          include-prerelease: true
-          dotnet-version: |
-            7.0.x
-
-      - name: Test .NET 7.0 (pre-release)
-        run: ./build.sh test
-
   interop:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Add libzstd
         shell: bash
         run: sudo apt-get install -y libzstd-dev
 
       - name: Install .NET SDKs
-        uses: actions/setup-dotnet@v1
+        uses: actions/setup-dotnet@v3
         with:
           dotnet-version: |
             3.1.x
             5.0.x
             6.0.x
+            7.0.x
 
       - name: Cache Local Maven Repository
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

.github/workflows/test-lang-java.yml

@@ -125,6 +125,7 @@ jobs:
             3.1.x
             5.0.x
             6.0.x
+            7.0.x
 
       - name: Install Java Avro for Interop Test
         working-directory: .

.mvn/extensions.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+      xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
+    <extension>
+        <groupId>org.apache.maven.extensions</groupId>
+        <artifactId>maven-build-cache-extension</artifactId>
+        <version>1.0.0</version>
+    </extension>
+</extensions>

build.sh

@@ -56,6 +56,9 @@ DOCKER_BUILD_XTRA_ARGS=${DOCKER_BUILD_XTRA_ARGS-}
 # Override the docker image name used.
 DOCKER_IMAGE_NAME=${DOCKER_IMAGE_NAME-}
 
+# When building a docker container, these are the files that will sent and available.
+DOCKER_EXTRA_CONTEXT="lang/ruby/Gemfile lang/ruby/avro.gemspec lang/ruby/Manifest share/VERSION.txt"
+
 usage() {
   echo "Usage: $0 {lint|test|dist|sign|clean|veryclean|docker [--args \"docker-args\"]|rat|githooks|docker-test}"
   exit 1
@@ -300,8 +303,9 @@ do
         echo "RUN getent group $GROUP_ID || groupadd -g $GROUP_ID $USER_NAME"
         echo "RUN getent passwd $USER_ID || useradd -g $GROUP_ID -u $USER_ID -k /root -m $USER_NAME"
       } > Dockerfile
+      # Include the ruby gemspec for preinstallation.
       # shellcheck disable=SC2086
-      tar -cf- lang/ruby/Gemfile Dockerfile | docker build $DOCKER_BUILD_XTRA_ARGS -t "$DOCKER_IMAGE_NAME" -
+      tar -cf- Dockerfile $DOCKER_EXTRA_CONTEXT | docker build $DOCKER_BUILD_XTRA_ARGS -t "$DOCKER_IMAGE_NAME" -
       rm Dockerfile
       # By mapping the .m2 directory you can do an mvn install from
       # within the container and use the result on your normal
@@ -336,7 +340,7 @@ do
       ;;
 
     docker-test)
-      tar -cf- share/docker/Dockerfile lang/ruby/Gemfile |
+      tar -cf- share/docker/Dockerfile $DOCKER_EXTRA_CONTEXT |
         docker build -t avro-test -f share/docker/Dockerfile -
       docker run --rm -v "${PWD}:/avro${DOCKER_MOUNT_FLAG}" --env "JAVA=${JAVA:-8}" avro-test /avro/share/docker/run-tests.sh
       ;;

lang/c++/api/Reader.hh

@@ -84,7 +84,7 @@ public:
         union {
             double d;
             uint64_t i;
-        } v;
+        } v = { 0 };
         reader_.read(v.i);
         val = v.d;
     }

lang/c++/api/buffer/Buffer.hh

@@ -145,7 +145,7 @@ public:
      **/
 
     size_type wroteTo(size_type size) {
-        int wrote = 0;
+        size_type wrote = 0;
         if (size) {
             if (size > freeSpace()) {
                 throw std::length_error("Impossible to write more data than free space");

lang/c/tests/test_avro_commons_schema.c

@@ -104,13 +104,16 @@ static void read_data(const char *dirpath, avro_schema_t schema) {
     fprintf(stdout, "\nExit run test OK => %d records", records_read);
     remove("./copy.avro");
     fflush(stdout);
+    avro_file_reader_close(reader);
+    avro_file_writer_close(writer);
 }
 
 static void run_tests(const char *dirpath)
 {
     fprintf(stdout, "\nRun test for path '%s'", dirpath);
     avro_schema_t schema = read_common_schema_test(dirpath);
     read_data(dirpath, schema);
+    avro_schema_decref(schema);
 }
 
 

lang/csharp/README.md

@@ -17,20 +17,20 @@ Install-Package Apache.Avro
 
 ## Project Target Frameworks
 
-| Project             | Published to nuget.org     | Type       | .NET Standard 2.0  | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0  | .NET 6.0  |
-|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|
-| Avro.main           | Apache.Avro                | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.File.Snappy    | Apache.Avro.File.Snappy    | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.File.BZip2     | Apache.Avro.File.BZip2     | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.File.XZ        | Apache.Avro.File.XZ        | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.codegen        | Apache.Avro.Tools          |  Exe        |                    |                   | ✔️            |✔️        |✔️        |
-| Avro.ipc            |                            | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.ipc.test       |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |
-| Avro.msbuild        |                            | Library    | ✔️                 | ✔️               |               |           |           |
-| Avro.perf           |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |
-| Avro.test           |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |
-| Avro.benchmark      |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |
+| Project             | Published to nuget.org     | Type       | .NET Standard 2.0  | .NET Standard 2.1 | .NET Core 3.1 | .NET 5.0  | .NET 6.0  | .NET 7.0  |
+|:-------------------:|:--------------------------:|:----------:|:------------------:|:-----------------:|:-------------:|:---------:|:---------:|:---------:|
+| Avro.main           | Apache.Avro                | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.File.Snappy    | Apache.Avro.File.Snappy    | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.File.BZip2     | Apache.Avro.File.BZip2     | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.File.XZ        | Apache.Avro.File.XZ        | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.File.Zstandard | Apache.Avro.File.Zstandard | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.codegen        | Apache.Avro.Tools          |  Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
+| Avro.ipc            |                            | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.ipc.test       |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |
+| Avro.msbuild        |                            | Library    | ✔️                 | ✔️               |               |           |           |           |
+| Avro.perf           |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
+| Avro.test           |                            | Unit Tests |                    |                   | ✔️            |✔️        |✔️        |✔️        |
+| Avro.benchmark      |                            | Exe        |                    |                   | ✔️            |✔️        |✔️        |✔️        |
 
 ## Dependency package version strategy
 

lang/csharp/build.sh

@@ -42,7 +42,7 @@ do
 
     perf)
       pushd ./src/apache/perf/
-      dotnet run --configuration Release --framework net6.0
+      dotnet run --configuration Release --framework net7.0
       ;;
 
     dist)
@@ -77,7 +77,7 @@ do
       ;;
 
     interop-data-generate)
-      dotnet run --project src/apache/test/Avro.test.csproj --framework net6.0 ../../share/test/schemas/interop.avsc ../../build/interop/data
+      dotnet run --project src/apache/test/Avro.test.csproj --framework net7.0 ../../share/test/schemas/interop.avsc ../../build/interop/data
       ;;
 
     interop-data-test)

lang/csharp/common.props

@@ -37,9 +37,7 @@
 
   <PropertyGroup Label="Target Frameworks">
     <!-- Exe -->
-    <!-- NOTE: .NET 7 is still in preview state, use it only if it is available to make sure the project is ready for it. When .NET 7 SDK is released preview, update this -->
-    <DefaultExeTargetFrameworks Condition="'$(NETCoreAppMaximumVersion)' != '7.0' or '$(_NETCoreSdkIsPreview)' == 'false'">netcoreapp3.1;net5.0;net6.0</DefaultExeTargetFrameworks>
-    <DefaultExeTargetFrameworks Condition="'$(NETCoreAppMaximumVersion)' == '7.0' and '$(_NETCoreSdkIsPreview)' == 'true'">net7.0</DefaultExeTargetFrameworks>
+    <DefaultExeTargetFrameworks>netcoreapp3.1;net5.0;net6.0;net7.0</DefaultExeTargetFrameworks>
     <!-- Library -->
     <DefaultLibraryTargetFrameworks>netstandard2.0;netstandard2.1</DefaultLibraryTargetFrameworks>
     <!-- Unit Tests -->
@@ -61,6 +59,12 @@
     <None Include="$(MSBuildThisFileDirectory)\..\..\doc\assets\icons\logo.png" Pack="true" Visible="false" PackagePath=""/>
   </ItemGroup>
 
+  <PropertyGroup>
+    <!-- Disable warning for EOL target frameworks -->
+    <CheckEolTargetFramework>false</CheckEolTargetFramework>
+    <SuppressTfmSupportBuildWarnings>true</SuppressTfmSupportBuildWarnings>
+  </PropertyGroup>
+
   <PropertyGroup>
     <IsTestProject Condition="'$(IsTestProject)' == ''">false</IsTestProject>
     <IsTestProject Condition="$(MSBuildProjectName.ToLower().EndsWith('.test')) or $(MSBuildProjectName.ToLower().StartsWith('.test'))">true</IsTestProject>

lang/csharp/src/apache/benchmark/Avro.benchmark.csproj

@@ -31,6 +31,12 @@
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <!-- Some schemas use lower cased names, which causes some class names being lower case only -->
+    <!-- e.g. The type name 'test' only contains lower-cased ascii characters. Such names may become reserved for the language. -->
+    <NoWarn>$(NoWarn);CS8981</NoWarn>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
   </ItemGroup>

lang/csharp/src/apache/benchmark/Program.cs

@@ -21,8 +21,8 @@ namespace Avro.Benchmark
 {
     public class Program
     {
-        // dotnet run -c Release -f net6.0
-        // dotnet run -c Release -f net6.0 --runtimes netcoreapp3.1 net5.0 net6.0
+        // dotnet run -c Release -f net7.0
+        // dotnet run -c Release -f net7.0 --runtimes netcoreapp3.1 net5.0 net6.0 net7.0
         public static void Main(string[] args)
         {
             BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);

lang/csharp/src/apache/main/IO/Encoder.cs

@@ -187,5 +187,10 @@ public interface Encoder
         /// <param name="start">Position within data where the contents start.</param>
         /// <param name="len">Number of bytes to write.</param>
         void WriteFixed(byte[] data, int start, int len);
+
+        /// <summary>
+        /// Flushes the encoder.
+        /// </summary>
+        void Flush();
     }
 }

lang/csharp/src/apache/main/IO/JsonEncoder.cs

@@ -28,7 +28,7 @@ namespace Avro.IO
     /// An <see cref="Encoder"/> for Avro's JSON data encoding.
     ///
     /// JsonEncoder buffers output, and data may not appear on the output until
-    /// <see cref="Flush()"/> is called.
+    /// <see cref="Encoder.Flush()"/> is called.
     ///
     /// JsonEncoder is not thread-safe.
     /// </summary>

lang/csharp/src/apache/test/IO/JsonCodecTests.cs

@@ -316,8 +316,8 @@ private byte[] fromJsonToAvro(string json, Schema schema)
             GenericDatumWriter<object> writer = new GenericDatumWriter<object>(schema);
             MemoryStream output = new MemoryStream();
 
-            JsonDecoder decoder = new JsonDecoder(schema, json);
-            BinaryEncoder encoder = new BinaryEncoder(output);
+            Decoder decoder = new JsonDecoder(schema, json);
+            Encoder encoder = new BinaryEncoder(output);
 
             object datum = reader.Read(null, decoder);