Skip to content

Commit d57bd01

Browse files
committed
update README with ORC sample props
1 parent 90f3001 commit d57bd01

File tree

3 files changed

+18
-11
lines changed

3 files changed

+18
-11
lines changed

packages/aws-cdk-lib/aws-kinesisfirehose/README.md

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,11 +200,20 @@ const outputFormat = new ParquetOutputFormat({
200200

201201
### Output Format: ORC
202202

203-
Example creation of custom ORC OutputFormat
203+
Example creation of custom ORC OutputFormat, with all values changed from the defaults.
204204

205205
```ts
206206
const outputFormat = new OrcOutputFormat({
207-
// TODO: Props
207+
formatVersion: OrcFormatVersion.V0_11,
208+
blockSize: core.Size.mebibytes(256),
209+
compression: Compression.UNCOMPRESSED,
210+
bloomFilterColumns: ['columnA'],
211+
bloomFilterFalsePositiveProbability: 0.1,
212+
dictionaryKeyThreshold: 0.7,
213+
enablePadding: true,
214+
paddingTolerance: 0.2,
215+
rowIndexStride: 9000,
216+
stripeSize: core.Size.mebibytes(32),
208217
})
209218
```
210219

packages/aws-cdk-lib/aws-kinesisfirehose/lib/record-format/output.ts

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ export interface ParquetOutputFormatProps {
8585
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
8686
*/
8787
export class ParquetOutputFormat implements IOutputFormat {
88-
private static readonly VALID_COMPRESSIONS = [Compression.SNAPPY, Compression.UNCOMPRESSED, Compression.GZIP];
88+
private static readonly VALID_COMPRESSIONS = [Compression.SNAPPY, Compression.UNCOMPRESSED, Compression.GZIP].map(compression => compression.value);
8989

9090
public constructor(readonly props?: ParquetOutputFormatProps) {
9191
this.validateProps(props);
@@ -96,7 +96,7 @@ export class ParquetOutputFormat implements IOutputFormat {
9696
return;
9797
}
9898

99-
if (props.compression !== undefined && !ParquetOutputFormat.VALID_COMPRESSIONS.map(compression => compression.value).includes(props.compression.value)) {
99+
if (props.compression !== undefined && !ParquetOutputFormat.VALID_COMPRESSIONS.includes(props.compression.value)) {
100100
throw new core.UnscopedValidationError(`Compression ${props.compression} is invalid, it must be one of ${ParquetOutputFormat.VALID_COMPRESSIONS}`);
101101
}
102102

@@ -221,8 +221,8 @@ export interface OrcOutputFormatProps {
221221
*
222222
* The default value is 0.05, which means 5 percent of stripe size.
223223
*
224-
* For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block.
225-
* In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space.
224+
* For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block.
225+
* In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space.
226226
* This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task.
227227
*
228228
* Kinesis Data Firehose ignores this parameter when `EnablePadding` is `false` .
@@ -260,7 +260,7 @@ export interface OrcOutputFormatProps {
260260
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
261261
*/
262262
class OrcOutputFormat implements IOutputFormat {
263-
private static readonly VALID_COMPRESSIONS = [Compression.SNAPPY, Compression.UNCOMPRESSED, Compression.GZIP];
263+
private static readonly VALID_COMPRESSIONS = [Compression.SNAPPY, Compression.UNCOMPRESSED, Compression.GZIP].map(compression => compression.value);
264264

265265
public constructor(readonly props?: OrcOutputFormatProps) {
266266
this.validateProps(props);
@@ -275,7 +275,7 @@ class OrcOutputFormat implements IOutputFormat {
275275
return;
276276
}
277277

278-
if (props.compression !== undefined && !OrcOutputFormat.VALID_COMPRESSIONS.map(compression => compression.value).includes(props.compression.value)) {
278+
if (props.compression !== undefined && !OrcOutputFormat.VALID_COMPRESSIONS.includes(props.compression.value)) {
279279
throw new core.UnscopedValidationError(`Compression ${props.compression} is invalid, it must be one of ${OrcOutputFormat.VALID_COMPRESSIONS}`);
280280
}
281281

@@ -329,12 +329,10 @@ class OrcOutputFormat implements IOutputFormat {
329329
}
330330
}
331331

332-
333332
/**
334333
* Represents possible output formats when performing record data conversion.
335334
*/
336335
export class OutputFormat {
337-
338336
/**
339337
* Write output files in Parquet
340338
*/

packages/aws-cdk-lib/aws-kinesisfirehose/lib/record-format/schema.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ export interface SchemaFromCfnTableProps {
6666
* @see http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-schemaconfiguration.html#cfn-kinesisfirehose-deliverystream-schemaconfiguration-versionid
6767
* @default `LATEST`
6868
*/
69-
readonly versionId?: string,
69+
readonly versionId?: string;
7070
}
7171

7272
/**

0 commit comments

Comments
 (0)