From 333a819ba4aa89ba9de5dbfa9b718ac47a7fd679 Mon Sep 17 00:00:00 2001 From: Minhua Wu Date: Tue, 17 Nov 2015 17:39:25 -0500 Subject: [PATCH 1/4] modify WaveData::Read(std::istream &is) function to deal with data read in stream mode when we don't know its exact size in advance --- src/feat/wave-reader.cc | 102 +++++++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/src/feat/wave-reader.cc b/src/feat/wave-reader.cc index cb3f287fdd6..8fe7af5a4e1 100644 --- a/src/feat/wave-reader.cc +++ b/src/feat/wave-reader.cc @@ -227,49 +227,73 @@ void WaveData::Read(std::istream &is) { // we allow the size to be off by one, because there is a weirdness in the // format of RIFF files that means that the input may sometimes be padded // with 1 unused byte to make the total size even. - KALDI_ERR << "Expected " << riff_chunk_size << " bytes in RIFF chunk, but " - << "after first data block there will be " << riff_chunk_read - << " + " << data_chunk_size << " bytes " - << "(we do not support reading multiple data chunks)."; - } - - std::vector data_pointer_vec; - std::vector data_size_vec; - uint32 num_bytes_read = 0; - for (int32 remain_chunk_size = data_chunk_size; remain_chunk_size > 0; - remain_chunk_size -= kBlockSize) { - int32 this_block_size = remain_chunk_size; - if (kBlockSize < remain_chunk_size) - this_block_size = kBlockSize; - char *block_data_vec = new char[this_block_size]; - is.read(block_data_vec, this_block_size); - num_bytes_read += is.gcount(); - data_size_vec.push_back(is.gcount()); - data_pointer_vec.push_back(block_data_vec); - if (num_bytes_read < this_block_size) - break; - } - - std::vector chunk_data_vec(num_bytes_read); - uint32 data_address = 0; - for (int i = 0; i < data_pointer_vec.size(); i++) { - memcpy(&(chunk_data_vec[data_address]), data_pointer_vec[i], - data_size_vec[i]); - delete[] data_pointer_vec[i]; - data_address += data_size_vec[i]; + if ((riff_chunk_size == 0 && data_chunk_size == 0) + || (riff_chunk_size == 4294967295 && data_chunk_size == 4294967295)) + KALDI_WARN << "Read in " << riff_chunk_size << " bytes in RIFF chunk, " + << "Read in " << riff_chunk_read << " bytes in data chunk." + << "maybe data is read in stream mode" + << "so that we don't know exact size in advance"; + else + KALDI_ERR << "Expected " << riff_chunk_size << " bytes in RIFF chunk, but " + << "after first data block there will be " << riff_chunk_read + << " + " << data_chunk_size << " bytes " + << "(we do not support reading multiple data chunks)."; } + std::vector chunk_data_vec; char *data_ptr = &(chunk_data_vec[0]); - if (num_bytes_read == 0 && num_bytes_read != data_chunk_size) { - KALDI_ERR << "WaveData: failed to read data chunk (read no bytes)"; - } else if (num_bytes_read != data_chunk_size) { - KALDI_ASSERT(num_bytes_read < data_chunk_size); - KALDI_WARN << "Read fewer bytes than specified in the header: " - << num_bytes_read << " < " << data_chunk_size; + uint32 num_bytes_read = 0; + if (!((riff_chunk_size == 0 && data_chunk_size == 0) + || (riff_chunk_size == 4294967295 && data_chunk_size == 4294967295))) { + std::vector data_pointer_vec; + std::vector data_size_vec; + num_bytes_read = 0; + for (int32 remain_chunk_size = data_chunk_size; remain_chunk_size > 0; + remain_chunk_size -= kBlockSize) { + int32 this_block_size = remain_chunk_size; + if (kBlockSize < remain_chunk_size) + this_block_size = kBlockSize; + char *block_data_vec = new char[this_block_size]; + is.read(block_data_vec, this_block_size); + num_bytes_read += is.gcount(); + data_size_vec.push_back(is.gcount()); + data_pointer_vec.push_back(block_data_vec); + if (num_bytes_read < this_block_size) + break; + } + chunk_data_vec.resize(num_bytes_read); + uint32 data_address = 0; + for (int i = 0; i < data_pointer_vec.size(); i++) { + KALDI_ASSERT(data_address + data_size_vec[i] <= chunk_data_vec.size()); + memcpy(&(chunk_data_vec[data_address]), data_pointer_vec[i], data_size_vec[i]); + delete[] data_pointer_vec[i]; + data_address += data_size_vec[i]; + } + + if (num_bytes_read == 0 && num_bytes_read != data_chunk_size) { + KALDI_ERR << "WaveData: failed to read data chunk (read no bytes)"; + } else if (num_bytes_read != data_chunk_size) { + KALDI_ASSERT(num_bytes_read < data_chunk_size); + KALDI_WARN << "Read fewer bytes than specified in the header: " + << num_bytes_read << " < " << data_chunk_size; + } + } else { + // when data are read in stream model, we don't know size in advance + // data_chunk_size and riff_chunk_size (Uint32) could be + // either ox00000000 or oxFFFFFFFF + // we need to read in the stream differently in this case + chunk_data_vec.clear(); + char this_data_byte; + num_bytes_read = 0; + // read it byte by byte since we don't know the exact size + while(is.get(this_data_byte)) { + chunk_data_vec.push_back(this_data_byte); + num_bytes_read++; + } } - - if (data_chunk_size == 0) - KALDI_ERR << "WaveData: empty file (no data)"; + if (num_bytes_read == 0) + KALDI_ERR << "WaveData: empty file (no data)"; + data_ptr = &(chunk_data_vec[0]); uint32 num_samp = num_bytes_read / block_align; data_.Resize(num_channels, num_samp); From 2b2be52dbf6cb5b9fe520d0911b785cc060890dc Mon Sep 17 00:00:00 2001 From: Minhua Wu Date: Wed, 18 Nov 2015 01:34:43 -0500 Subject: [PATCH 2/4] check is_stream_mode in advance; modify warning text format --- src/feat/wave-reader.cc | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/feat/wave-reader.cc b/src/feat/wave-reader.cc index 8fe7af5a4e1..eb391252b13 100644 --- a/src/feat/wave-reader.cc +++ b/src/feat/wave-reader.cc @@ -221,18 +221,28 @@ void WaveData::Read(std::istream &is) { uint32 data_chunk_size = ReadUint32(is, swap); riff_chunk_read += 4; + // check if data is read in stream mode + bool is_stream_mode = false; + if ((riff_chunk_size == 0 && data_chunk_size == 0) + || (riff_chunk_size == std::numeric_limits::max() + && data_chunk_size == std::numeric_limits::max())) { + // when data are read in stream model, we don't know chunk size in advance + // data_chunk_size and riff_chunk_size (Uint32) could be + // either ox00000000 or oxFFFFFFFF + is_stream_mode = true; + } + if (std::abs(static_cast(riff_chunk_read) + static_cast(data_chunk_size) - static_cast(riff_chunk_size)) > 1) { // we allow the size to be off by one, because there is a weirdness in the // format of RIFF files that means that the input may sometimes be padded // with 1 unused byte to make the total size even. - if ((riff_chunk_size == 0 && data_chunk_size == 0) - || (riff_chunk_size == 4294967295 && data_chunk_size == 4294967295)) - KALDI_WARN << "Read in " << riff_chunk_size << " bytes in RIFF chunk, " - << "Read in " << riff_chunk_read << " bytes in data chunk." - << "maybe data is read in stream mode" - << "so that we don't know exact size in advance"; + if (is_stream_mode) + KALDI_WARN << "Read in RIFF chunk size: " << riff_chunk_size + << ", Read in data chunk size: " << data_chunk_size + << ". Maybe data is read in stream mode " + << "so that we don't know exact size in advance."; else KALDI_ERR << "Expected " << riff_chunk_size << " bytes in RIFF chunk, but " << "after first data block there will be " << riff_chunk_read @@ -241,10 +251,8 @@ void WaveData::Read(std::istream &is) { } std::vector chunk_data_vec; - char *data_ptr = &(chunk_data_vec[0]); uint32 num_bytes_read = 0; - if (!((riff_chunk_size == 0 && data_chunk_size == 0) - || (riff_chunk_size == 4294967295 && data_chunk_size == 4294967295))) { + if (!is_stream_mode) { std::vector data_pointer_vec; std::vector data_size_vec; num_bytes_read = 0; @@ -293,7 +301,7 @@ void WaveData::Read(std::istream &is) { } if (num_bytes_read == 0) KALDI_ERR << "WaveData: empty file (no data)"; - data_ptr = &(chunk_data_vec[0]); + char *data_ptr = &(chunk_data_vec[0]); uint32 num_samp = num_bytes_read / block_align; data_.Resize(num_channels, num_samp); From 3d9b40026642bafd499b7e0c97f98414627a8f89 Mon Sep 17 00:00:00 2001 From: Minhua Wu Date: Thu, 19 Nov 2015 01:22:27 -0500 Subject: [PATCH 3/4] read sizable chunks from the stream, and push them into the chunk_data_vec --- src/feat/wave-reader.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/feat/wave-reader.cc b/src/feat/wave-reader.cc index eb391252b13..6e992387324 100644 --- a/src/feat/wave-reader.cc +++ b/src/feat/wave-reader.cc @@ -291,12 +291,13 @@ void WaveData::Read(std::istream &is) { // either ox00000000 or oxFFFFFFFF // we need to read in the stream differently in this case chunk_data_vec.clear(); - char this_data_byte; num_bytes_read = 0; - // read it byte by byte since we don't know the exact size - while(is.get(this_data_byte)) { - chunk_data_vec.push_back(this_data_byte); - num_bytes_read++; + while (is.good()) { + char buffer[256]; + is.read(buffer, sizeof buffer); + std::copy(buffer, buffer + is.gcount(), + std::back_inserter(chunk_data_vec)); + num_bytes_read += is.gcount(); } } if (num_bytes_read == 0) From c700f9181b5e56ae33d22180d678ff638f579f8c Mon Sep 17 00:00:00 2001 From: Minhua Wu Date: Sat, 21 Nov 2015 00:19:41 -0500 Subject: [PATCH 4/4] rewrite original wave read code --- src/feat/wave-reader.cc | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/src/feat/wave-reader.cc b/src/feat/wave-reader.cc index 6e992387324..94c37f02b58 100644 --- a/src/feat/wave-reader.cc +++ b/src/feat/wave-reader.cc @@ -253,38 +253,10 @@ void WaveData::Read(std::istream &is) { std::vector chunk_data_vec; uint32 num_bytes_read = 0; if (!is_stream_mode) { - std::vector data_pointer_vec; - std::vector data_size_vec; - num_bytes_read = 0; - for (int32 remain_chunk_size = data_chunk_size; remain_chunk_size > 0; - remain_chunk_size -= kBlockSize) { - int32 this_block_size = remain_chunk_size; - if (kBlockSize < remain_chunk_size) - this_block_size = kBlockSize; - char *block_data_vec = new char[this_block_size]; - is.read(block_data_vec, this_block_size); - num_bytes_read += is.gcount(); - data_size_vec.push_back(is.gcount()); - data_pointer_vec.push_back(block_data_vec); - if (num_bytes_read < this_block_size) - break; - } - chunk_data_vec.resize(num_bytes_read); - uint32 data_address = 0; - for (int i = 0; i < data_pointer_vec.size(); i++) { - KALDI_ASSERT(data_address + data_size_vec[i] <= chunk_data_vec.size()); - memcpy(&(chunk_data_vec[data_address]), data_pointer_vec[i], data_size_vec[i]); - delete[] data_pointer_vec[i]; - data_address += data_size_vec[i]; - } - - if (num_bytes_read == 0 && num_bytes_read != data_chunk_size) { - KALDI_ERR << "WaveData: failed to read data chunk (read no bytes)"; - } else if (num_bytes_read != data_chunk_size) { - KALDI_ASSERT(num_bytes_read < data_chunk_size); - KALDI_WARN << "Read fewer bytes than specified in the header: " - << num_bytes_read << " < " << data_chunk_size; - } + chunk_data_vec.resize(data_chunk_size); + is.read(&chunk_data_vec[0], data_chunk_size); + num_bytes_read = is.gcount(); + KALDI_ASSERT(num_bytes_read == data_chunk_size); } else { // when data are read in stream model, we don't know size in advance // data_chunk_size and riff_chunk_size (Uint32) could be @@ -293,7 +265,7 @@ void WaveData::Read(std::istream &is) { chunk_data_vec.clear(); num_bytes_read = 0; while (is.good()) { - char buffer[256]; + char buffer[kBlockSize]; is.read(buffer, sizeof buffer); std::copy(buffer, buffer + is.gcount(), std::back_inserter(chunk_data_vec)); @@ -301,7 +273,7 @@ void WaveData::Read(std::istream &is) { } } if (num_bytes_read == 0) - KALDI_ERR << "WaveData: empty file (no data)"; + KALDI_ERR << "WaveData: empty file (no data)"; char *data_ptr = &(chunk_data_vec[0]); uint32 num_samp = num_bytes_read / block_align;