Skip to content

Revert "AVRO-3684: [Java] testAppendStream test" #2108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nielsbasjes merged 1 commit into from
Feb 18, 2023
Merged

Revert "AVRO-3684: [Java] testAppendStream test" #2108

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 0 additions & 64 deletions lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
Expand All @@ -37,15 +36,11 @@
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.file.FileReader;
import org.apache.avro.file.SeekableFileInput;
import org.apache.avro.file.SyncableFileOutputStream;
import org.apache.avro.file.Syncable;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.RandomData;

import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -107,7 +102,6 @@ public void runTestsInOrder(CodecFactory codec) throws Exception {
testReadWithHeader(codec);
testFSync(codec, false);
testFSync(codec, true);
testAppendStream(codec);
}

private void testGenericWrite(CodecFactory codec) throws IOException {
Expand Down Expand Up @@ -339,64 +333,6 @@ private void testFSync(CodecFactory codec, boolean useFile) throws IOException {
}
}

private void testAppendStream(CodecFactory codec) throws IOException {
File file = makeFile(codec);

DatumWriter<Object> datumWriter = new SpecificDatumWriter<>();

// write COUNT objects to datafile
try (DataFileWriter<Object> writer = new DataFileWriter<>(datumWriter)) {
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {

SyncableFileOutputStream fileOutputStream = new SyncableFileOutputStream(raf.getFD());

writer.create(SCHEMA, fileOutputStream);
for (Object datum : new RandomData(SCHEMA, COUNT, SEED)) {
writer.append(datum);
writer.flush();
}
}
}

// append to existing file
try (DataFileWriter<Object> writer = new DataFileWriter<>(datumWriter)) {
try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
try (RandomAccessFile rif = new RandomAccessFile(file, "r")) {

SyncableFileOutputStream fileOutputStream = new SyncableFileOutputStream(raf.getFD());

// seek to end
fileOutputStream.getChannel().position(fileOutputStream.getChannel().size());

// append to existing fileStream
try (SeekableFileInput in = new SeekableFileInput(rif.getFD())) {
writer.appendTo(in, fileOutputStream);
}

for (Object datum : new RandomData(SCHEMA, COUNT, SEED + 1)) {
writer.append(datum);
writer.flush();
}
}
}
}

// verify objects in file
long recordCounter = 0;

try (RandomAccessFile raf = new RandomAccessFile(file, "r")) {
SeekableFileInput in = new SeekableFileInput(raf.getFD());
DatumReader<Object> datumReader = new SpecificDatumReader<>();
try (FileReader<? extends Object> reader = DataFileReader.openReader(in, datumReader)) {
while (reader.hasNext()) {
reader.next();
recordCounter++;
}
}
}
assertEquals(COUNT * 2, recordCounter);
}

static void readFile(File f, DatumReader<? extends Object> datumReader) throws IOException {
try (FileReader<? extends Object> reader = DataFileReader.openReader(f, datumReader)) {
for (Object datum : reader) {
Expand Down