diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt index 5d107c1e50fd8..05896a4d69b47 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt @@ -6,97 +6,109 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 20802 20802 0 4.8 208.0 1.0X -before 1582, noop 10728 10728 0 9.3 107.3 1.9X -after 1582, rebase off 32924 32924 0 3.0 329.2 0.6X -after 1582, rebase on 32627 32627 0 3.1 326.3 0.6X -before 1582, rebase off 21576 21576 0 4.6 215.8 1.0X -before 1582, rebase on 23115 23115 0 4.3 231.2 0.9X +after 1582, noop 20023 20023 0 5.0 200.2 1.0X +before 1582, noop 10729 10729 0 9.3 107.3 1.9X +after 1582, rebase EXCEPTION 31834 31834 0 3.1 318.3 0.6X +after 1582, rebase LEGACY 31997 31997 0 3.1 320.0 0.6X +after 1582, rebase CORRECTED 31712 31712 0 3.2 317.1 0.6X +before 1582, rebase LEGACY 23663 23663 0 4.2 236.6 0.8X +before 1582, rebase CORRECTED 22749 22749 0 4.4 227.5 0.9X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase off 12880 12984 178 7.8 128.8 1.0X -after 1582, vec off, rebase on 13118 13255 174 7.6 131.2 1.0X -after 1582, vec on, rebase off 3645 3698 76 27.4 36.4 3.5X -after 1582, vec on, rebase on 3709 3727 15 27.0 37.1 3.5X -before 1582, vec off, rebase off 13014 13051 36 7.7 130.1 1.0X -before 1582, vec off, rebase on 14195 14242 48 7.0 142.0 0.9X -before 1582, vec on, rebase off 3680 3773 92 27.2 36.8 3.5X -before 1582, vec on, rebase on 4310 4381 87 23.2 43.1 3.0X +after 1582, vec off, rebase EXCEPTION 12984 13262 257 7.7 129.8 1.0X +after 1582, vec off, rebase LEGACY 13278 13330 50 7.5 132.8 1.0X +after 1582, vec off, rebase CORRECTED 13202 13255 50 7.6 132.0 1.0X +after 1582, vec on, rebase EXCEPTION 3823 3853 40 26.2 38.2 3.4X +after 1582, vec on, rebase LEGACY 3846 3876 27 26.0 38.5 3.4X +after 1582, vec on, rebase CORRECTED 3775 3838 62 26.5 37.7 3.4X +before 1582, vec off, rebase LEGACY 13671 13692 26 7.3 136.7 0.9X +before 1582, vec off, rebase CORRECTED 13387 13476 106 7.5 133.9 1.0X +before 1582, vec on, rebase LEGACY 4477 4484 7 22.3 44.8 2.9X +before 1582, vec on, rebase CORRECTED 3729 3773 50 26.8 37.3 3.5X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 3026 3026 0 33.1 30.3 1.0X -before 1900, noop 2995 2995 0 33.4 30.0 1.0X -after 1900, rebase off 24294 24294 0 4.1 242.9 0.1X -after 1900, rebase on 24480 24480 0 4.1 244.8 0.1X -before 1900, rebase off 31120 31120 0 3.2 311.2 0.1X -before 1900, rebase on 31201 31201 0 3.2 312.0 0.1X +after 1900, noop 3020 3020 0 33.1 30.2 1.0X +before 1900, noop 3013 3013 0 33.2 30.1 1.0X +after 1900, rebase EXCEPTION 28796 28796 0 3.5 288.0 0.1X +after 1900, rebase LEGACY 28869 28869 0 3.5 288.7 0.1X +after 1900, rebase CORRECTED 28522 28522 0 3.5 285.2 0.1X +before 1900, rebase LEGACY 30594 30594 0 3.3 305.9 0.1X +before 1900, rebase CORRECTED 30743 30743 0 3.3 307.4 0.1X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 18283 18309 39 5.5 182.8 1.0X -after 1900, vec off, rebase on 18235 18269 53 5.5 182.4 1.0X -after 1900, vec on, rebase off 9563 9589 27 10.5 95.6 1.9X -after 1900, vec on, rebase on 9463 9554 81 10.6 94.6 1.9X -before 1900, vec off, rebase off 21377 21469 118 4.7 213.8 0.9X -before 1900, vec off, rebase on 21265 21422 156 4.7 212.7 0.9X -before 1900, vec on, rebase off 12481 12524 46 8.0 124.8 1.5X -before 1900, vec on, rebase on 12360 12482 105 8.1 123.6 1.5X +after 1900, vec off, rebase EXCEPTION 19325 19468 135 5.2 193.3 1.0X +after 1900, vec off, rebase LEGACY 19568 19602 30 5.1 195.7 1.0X +after 1900, vec off, rebase CORRECTED 19532 19538 6 5.1 195.3 1.0X +after 1900, vec on, rebase EXCEPTION 9884 9990 94 10.1 98.8 2.0X +after 1900, vec on, rebase LEGACY 9933 9985 49 10.1 99.3 1.9X +after 1900, vec on, rebase CORRECTED 9967 10043 76 10.0 99.7 1.9X +before 1900, vec off, rebase LEGACY 24162 24198 37 4.1 241.6 0.8X +before 1900, vec off, rebase CORRECTED 24034 24056 20 4.2 240.3 0.8X +before 1900, vec on, rebase LEGACY 12548 12625 72 8.0 125.5 1.5X +before 1900, vec on, rebase CORRECTED 12580 12660 115 7.9 125.8 1.5X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2984 2984 0 33.5 29.8 1.0X -before 1900, noop 3003 3003 0 33.3 30.0 1.0X -after 1900, rebase off 15814 15814 0 6.3 158.1 0.2X -after 1900, rebase on 16250 16250 0 6.2 162.5 0.2X -before 1900, rebase off 16026 16026 0 6.2 160.3 0.2X -before 1900, rebase on 19735 19735 0 5.1 197.3 0.2X +after 1900, noop 3159 3159 0 31.7 31.6 1.0X +before 1900, noop 3038 3038 0 32.9 30.4 1.0X +after 1900, rebase EXCEPTION 16885 16885 0 5.9 168.8 0.2X +after 1900, rebase LEGACY 17171 17171 0 5.8 171.7 0.2X +after 1900, rebase CORRECTED 17353 17353 0 5.8 173.5 0.2X +before 1900, rebase LEGACY 20579 20579 0 4.9 205.8 0.2X +before 1900, rebase CORRECTED 17544 17544 0 5.7 175.4 0.2X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 15292 15351 57 6.5 152.9 1.0X -after 1900, vec off, rebase on 15753 15886 173 6.3 157.5 1.0X -after 1900, vec on, rebase off 4879 4923 52 20.5 48.8 3.1X -after 1900, vec on, rebase on 5018 5038 18 19.9 50.2 3.0X -before 1900, vec off, rebase off 15257 15311 53 6.6 152.6 1.0X -before 1900, vec off, rebase on 18459 18537 90 5.4 184.6 0.8X -before 1900, vec on, rebase off 4929 4946 15 20.3 49.3 3.1X -before 1900, vec on, rebase on 8254 8339 93 12.1 82.5 1.9X +after 1900, vec off, rebase EXCEPTION 16304 16345 58 6.1 163.0 1.0X +after 1900, vec off, rebase LEGACY 16503 16585 75 6.1 165.0 1.0X +after 1900, vec off, rebase CORRECTED 16413 16463 44 6.1 164.1 1.0X +after 1900, vec on, rebase EXCEPTION 5017 5034 29 19.9 50.2 3.2X +after 1900, vec on, rebase LEGACY 5060 5094 30 19.8 50.6 3.2X +after 1900, vec on, rebase CORRECTED 4969 4971 1 20.1 49.7 3.3X +before 1900, vec off, rebase LEGACY 19767 20001 203 5.1 197.7 0.8X +before 1900, vec off, rebase CORRECTED 16421 16465 38 6.1 164.2 1.0X +before 1900, vec on, rebase LEGACY 8535 8608 64 11.7 85.4 1.9X +before 1900, vec on, rebase CORRECTED 5044 5077 32 19.8 50.4 3.2X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2987 2987 0 33.5 29.9 1.0X -before 1900, noop 3002 3002 0 33.3 30.0 1.0X -after 1900, rebase off 15215 15215 0 6.6 152.1 0.2X -after 1900, rebase on 15577 15577 0 6.4 155.8 0.2X -before 1900, rebase off 15505 15505 0 6.4 155.1 0.2X -before 1900, rebase on 19143 19143 0 5.2 191.4 0.2X +after 1900, noop 2995 2995 0 33.4 29.9 1.0X +before 1900, noop 2981 2981 0 33.5 29.8 1.0X +after 1900, rebase EXCEPTION 16196 16196 0 6.2 162.0 0.2X +after 1900, rebase LEGACY 16550 16550 0 6.0 165.5 0.2X +after 1900, rebase CORRECTED 16908 16908 0 5.9 169.1 0.2X +before 1900, rebase LEGACY 20087 20087 0 5.0 200.9 0.1X +before 1900, rebase CORRECTED 17171 17171 0 5.8 171.7 0.2X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 15330 15436 113 6.5 153.3 1.0X -after 1900, vec off, rebase on 15515 15549 30 6.4 155.1 1.0X -after 1900, vec on, rebase off 6056 6074 19 16.5 60.6 2.5X -after 1900, vec on, rebase on 6376 6390 14 15.7 63.8 2.4X -before 1900, vec off, rebase off 15490 15523 36 6.5 154.9 1.0X -before 1900, vec off, rebase on 18613 18685 118 5.4 186.1 0.8X -before 1900, vec on, rebase off 6065 6109 41 16.5 60.6 2.5X -before 1900, vec on, rebase on 9052 9082 32 11.0 90.5 1.7X +after 1900, vec off, rebase EXCEPTION 16688 16787 88 6.0 166.9 1.0X +after 1900, vec off, rebase LEGACY 17383 17462 73 5.8 173.8 1.0X +after 1900, vec off, rebase CORRECTED 17317 17329 11 5.8 173.2 1.0X +after 1900, vec on, rebase EXCEPTION 6342 6348 6 15.8 63.4 2.6X +after 1900, vec on, rebase LEGACY 6500 6521 18 15.4 65.0 2.6X +after 1900, vec on, rebase CORRECTED 6164 6172 11 16.2 61.6 2.7X +before 1900, vec off, rebase LEGACY 20575 20665 81 4.9 205.7 0.8X +before 1900, vec off, rebase CORRECTED 17239 17290 61 5.8 172.4 1.0X +before 1900, vec on, rebase LEGACY 9310 9373 60 10.7 93.1 1.8X +before 1900, vec on, rebase CORRECTED 6091 6105 16 16.4 60.9 2.7X ================================================================================================ @@ -107,36 +119,36 @@ OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-106 Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 20653 20653 0 4.8 206.5 1.0X -before 1582, noop 10707 10707 0 9.3 107.1 1.9X -after 1582 28288 28288 0 3.5 282.9 0.7X -before 1582 19196 19196 0 5.2 192.0 1.1X +after 1582, noop 19583 19583 0 5.1 195.8 1.0X +before 1582, noop 10711 10711 0 9.3 107.1 1.8X +after 1582 27864 27864 0 3.6 278.6 0.7X +before 1582 19648 19648 0 5.1 196.5 1.0X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 10596 10621 37 9.4 106.0 1.0X -after 1582, vec on 3886 3938 61 25.7 38.9 2.7X -before 1582, vec off 10955 10984 26 9.1 109.6 1.0X -before 1582, vec on 4236 4258 24 23.6 42.4 2.5X +after 1582, vec off 10383 10560 192 9.6 103.8 1.0X +after 1582, vec on 3844 3864 33 26.0 38.4 2.7X +before 1582, vec off 10867 10916 48 9.2 108.7 1.0X +before 1582, vec on 4158 4170 12 24.0 41.6 2.5X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2988 2988 0 33.5 29.9 1.0X -before 1900, noop 3007 3007 0 33.3 30.1 1.0X -after 1900 18082 18082 0 5.5 180.8 0.2X -before 1900 22669 22669 0 4.4 226.7 0.1X +after 1900, noop 2989 2989 0 33.5 29.9 1.0X +before 1900, noop 3000 3000 0 33.3 30.0 1.0X +after 1900 19426 19426 0 5.1 194.3 0.2X +before 1900 23282 23282 0 4.3 232.8 0.1X OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 12029 12035 9 8.3 120.3 1.0X -after 1900, vec on 5194 5197 3 19.3 51.9 2.3X -before 1900, vec off 14853 14875 23 6.7 148.5 0.8X -before 1900, vec on 7797 7836 60 12.8 78.0 1.5X +after 1900, vec off 12089 12102 15 8.3 120.9 1.0X +after 1900, vec on 5210 5325 100 19.2 52.1 2.3X +before 1900, vec off 15320 15373 46 6.5 153.2 0.8X +before 1900, vec on 7937 7970 48 12.6 79.4 1.5X diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt index f2af3cc000f04..3e94d6c6fcfa7 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt @@ -6,97 +6,109 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 23567 23567 0 4.2 235.7 1.0X -before 1582, noop 10570 10570 0 9.5 105.7 2.2X -after 1582, rebase off 35335 35335 0 2.8 353.3 0.7X -after 1582, rebase on 35645 35645 0 2.8 356.5 0.7X -before 1582, rebase off 21824 21824 0 4.6 218.2 1.1X -before 1582, rebase on 22532 22532 0 4.4 225.3 1.0X +after 1582, noop 23300 23300 0 4.3 233.0 1.0X +before 1582, noop 10585 10585 0 9.4 105.9 2.2X +after 1582, rebase EXCEPTION 35215 35215 0 2.8 352.1 0.7X +after 1582, rebase LEGACY 34927 34927 0 2.9 349.3 0.7X +after 1582, rebase CORRECTED 35479 35479 0 2.8 354.8 0.7X +before 1582, rebase LEGACY 22767 22767 0 4.4 227.7 1.0X +before 1582, rebase CORRECTED 22527 22527 0 4.4 225.3 1.0X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase off 13194 13266 81 7.6 131.9 1.0X -after 1582, vec off, rebase on 13402 13466 89 7.5 134.0 1.0X -after 1582, vec on, rebase off 3627 3657 29 27.6 36.3 3.6X -after 1582, vec on, rebase on 3818 3839 26 26.2 38.2 3.5X -before 1582, vec off, rebase off 13075 13146 115 7.6 130.7 1.0X -before 1582, vec off, rebase on 13794 13804 13 7.2 137.9 1.0X -before 1582, vec on, rebase off 3655 3675 21 27.4 36.6 3.6X -before 1582, vec on, rebase on 4579 4634 72 21.8 45.8 2.9X +after 1582, vec off, rebase EXCEPTION 13480 13577 94 7.4 134.8 1.0X +after 1582, vec off, rebase LEGACY 13466 13586 118 7.4 134.7 1.0X +after 1582, vec off, rebase CORRECTED 13526 13558 41 7.4 135.3 1.0X +after 1582, vec on, rebase EXCEPTION 3759 3778 28 26.6 37.6 3.6X +after 1582, vec on, rebase LEGACY 3957 4004 57 25.3 39.6 3.4X +after 1582, vec on, rebase CORRECTED 3739 3755 25 26.7 37.4 3.6X +before 1582, vec off, rebase LEGACY 13986 14038 67 7.1 139.9 1.0X +before 1582, vec off, rebase CORRECTED 13453 13491 49 7.4 134.5 1.0X +before 1582, vec on, rebase LEGACY 4716 4724 10 21.2 47.2 2.9X +before 1582, vec on, rebase CORRECTED 3701 3750 50 27.0 37.0 3.6X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2671 2671 0 37.4 26.7 1.0X -before 1900, noop 2685 2685 0 37.2 26.8 1.0X -after 1900, rebase off 23899 23899 0 4.2 239.0 0.1X -after 1900, rebase on 24030 24030 0 4.2 240.3 0.1X -before 1900, rebase off 30178 30178 0 3.3 301.8 0.1X -before 1900, rebase on 30127 30127 0 3.3 301.3 0.1X +after 1900, noop 2790 2790 0 35.8 27.9 1.0X +before 1900, noop 2812 2812 0 35.6 28.1 1.0X +after 1900, rebase EXCEPTION 24789 24789 0 4.0 247.9 0.1X +after 1900, rebase LEGACY 24539 24539 0 4.1 245.4 0.1X +after 1900, rebase CORRECTED 24543 24543 0 4.1 245.4 0.1X +before 1900, rebase LEGACY 30496 30496 0 3.3 305.0 0.1X +before 1900, rebase CORRECTED 30428 30428 0 3.3 304.3 0.1X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 16613 16685 75 6.0 166.1 1.0X -after 1900, vec off, rebase on 16487 16541 47 6.1 164.9 1.0X -after 1900, vec on, rebase off 8840 8870 49 11.3 88.4 1.9X -after 1900, vec on, rebase on 8795 8813 20 11.4 87.9 1.9X -before 1900, vec off, rebase off 20400 20441 62 4.9 204.0 0.8X -before 1900, vec off, rebase on 20430 20481 60 4.9 204.3 0.8X -before 1900, vec on, rebase off 12211 12290 73 8.2 122.1 1.4X -before 1900, vec on, rebase on 12231 12321 95 8.2 122.3 1.4X +after 1900, vec off, rebase EXCEPTION 17106 17192 75 5.8 171.1 1.0X +after 1900, vec off, rebase LEGACY 17273 17337 55 5.8 172.7 1.0X +after 1900, vec off, rebase CORRECTED 17073 17215 128 5.9 170.7 1.0X +after 1900, vec on, rebase EXCEPTION 8903 8976 117 11.2 89.0 1.9X +after 1900, vec on, rebase LEGACY 8793 8876 84 11.4 87.9 1.9X +after 1900, vec on, rebase CORRECTED 8820 8878 53 11.3 88.2 1.9X +before 1900, vec off, rebase LEGACY 20997 21069 82 4.8 210.0 0.8X +before 1900, vec off, rebase CORRECTED 20874 20946 90 4.8 208.7 0.8X +before 1900, vec on, rebase LEGACY 12024 12090 58 8.3 120.2 1.4X +before 1900, vec on, rebase CORRECTED 12020 12069 64 8.3 120.2 1.4X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2836 2836 0 35.3 28.4 1.0X -before 1900, noop 2812 2812 0 35.6 28.1 1.0X -after 1900, rebase off 15976 15976 0 6.3 159.8 0.2X -after 1900, rebase on 16197 16197 0 6.2 162.0 0.2X -before 1900, rebase off 16140 16140 0 6.2 161.4 0.2X -before 1900, rebase on 20410 20410 0 4.9 204.1 0.1X +after 1900, noop 2939 2939 0 34.0 29.4 1.0X +before 1900, noop 2917 2917 0 34.3 29.2 1.0X +after 1900, rebase EXCEPTION 15954 15954 0 6.3 159.5 0.2X +after 1900, rebase LEGACY 16402 16402 0 6.1 164.0 0.2X +after 1900, rebase CORRECTED 16541 16541 0 6.0 165.4 0.2X +before 1900, rebase LEGACY 20500 20500 0 4.9 205.0 0.1X +before 1900, rebase CORRECTED 16764 16764 0 6.0 167.6 0.2X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 15297 15324 40 6.5 153.0 1.0X -after 1900, vec off, rebase on 15771 15832 59 6.3 157.7 1.0X -after 1900, vec on, rebase off 4922 4949 32 20.3 49.2 3.1X -after 1900, vec on, rebase on 5392 5411 17 18.5 53.9 2.8X -before 1900, vec off, rebase off 15227 15385 141 6.6 152.3 1.0X -before 1900, vec off, rebase on 19611 19658 41 5.1 196.1 0.8X -before 1900, vec on, rebase off 4965 5013 54 20.1 49.6 3.1X -before 1900, vec on, rebase on 9847 9873 43 10.2 98.5 1.6X +after 1900, vec off, rebase EXCEPTION 15607 15655 81 6.4 156.1 1.0X +after 1900, vec off, rebase LEGACY 15616 15676 54 6.4 156.2 1.0X +after 1900, vec off, rebase CORRECTED 15634 15732 108 6.4 156.3 1.0X +after 1900, vec on, rebase EXCEPTION 5041 5057 16 19.8 50.4 3.1X +after 1900, vec on, rebase LEGACY 5516 5539 29 18.1 55.2 2.8X +after 1900, vec on, rebase CORRECTED 5087 5104 28 19.7 50.9 3.1X +before 1900, vec off, rebase LEGACY 19262 19338 79 5.2 192.6 0.8X +before 1900, vec off, rebase CORRECTED 15718 15755 53 6.4 157.2 1.0X +before 1900, vec on, rebase LEGACY 10147 10240 114 9.9 101.5 1.5X +before 1900, vec on, rebase CORRECTED 5062 5080 21 19.8 50.6 3.1X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2818 2818 0 35.5 28.2 1.0X -before 1900, noop 2805 2805 0 35.6 28.1 1.0X -after 1900, rebase off 15182 15182 0 6.6 151.8 0.2X -after 1900, rebase on 15614 15614 0 6.4 156.1 0.2X -before 1900, rebase off 15404 15404 0 6.5 154.0 0.2X -before 1900, rebase on 19747 19747 0 5.1 197.5 0.1X +after 1900, noop 2915 2915 0 34.3 29.2 1.0X +before 1900, noop 2894 2894 0 34.6 28.9 1.0X +after 1900, rebase EXCEPTION 15545 15545 0 6.4 155.4 0.2X +after 1900, rebase LEGACY 15840 15840 0 6.3 158.4 0.2X +after 1900, rebase CORRECTED 16324 16324 0 6.1 163.2 0.2X +before 1900, rebase LEGACY 20359 20359 0 4.9 203.6 0.1X +before 1900, rebase CORRECTED 16292 16292 0 6.1 162.9 0.2X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase off 15622 15649 24 6.4 156.2 1.0X -after 1900, vec off, rebase on 15572 15677 119 6.4 155.7 1.0X -after 1900, vec on, rebase off 6345 6358 15 15.8 63.5 2.5X -after 1900, vec on, rebase on 6780 6834 92 14.8 67.8 2.3X -before 1900, vec off, rebase off 15540 15584 38 6.4 155.4 1.0X -before 1900, vec off, rebase on 19590 19653 55 5.1 195.9 0.8X -before 1900, vec on, rebase off 6374 6381 10 15.7 63.7 2.5X -before 1900, vec on, rebase on 10530 10544 25 9.5 105.3 1.5X +after 1900, vec off, rebase EXCEPTION 15857 16015 223 6.3 158.6 1.0X +after 1900, vec off, rebase LEGACY 16174 16231 63 6.2 161.7 1.0X +after 1900, vec off, rebase CORRECTED 16353 16400 67 6.1 163.5 1.0X +after 1900, vec on, rebase EXCEPTION 6449 6459 9 15.5 64.5 2.5X +after 1900, vec on, rebase LEGACY 7028 7035 6 14.2 70.3 2.3X +after 1900, vec on, rebase CORRECTED 6585 6623 37 15.2 65.8 2.4X +before 1900, vec off, rebase LEGACY 19929 20027 95 5.0 199.3 0.8X +before 1900, vec off, rebase CORRECTED 16401 16451 49 6.1 164.0 1.0X +before 1900, vec on, rebase LEGACY 10517 10563 40 9.5 105.2 1.5X +before 1900, vec on, rebase CORRECTED 6659 6675 26 15.0 66.6 2.4X ================================================================================================ @@ -107,36 +119,36 @@ OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aw Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 23825 23825 0 4.2 238.2 1.0X -before 1582, noop 10501 10501 0 9.5 105.0 2.3X -after 1582 32134 32134 0 3.1 321.3 0.7X -before 1582 19947 19947 0 5.0 199.5 1.2X +after 1582, noop 22782 22782 0 4.4 227.8 1.0X +before 1582, noop 10555 10555 0 9.5 105.6 2.2X +after 1582 31497 31497 0 3.2 315.0 0.7X +before 1582 19803 19803 0 5.0 198.0 1.2X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 10034 10056 22 10.0 100.3 1.0X -after 1582, vec on 3664 3698 30 27.3 36.6 2.7X -before 1582, vec off 10472 10502 30 9.5 104.7 1.0X -before 1582, vec on 4052 4098 42 24.7 40.5 2.5X +after 1582, vec off 10180 10214 44 9.8 101.8 1.0X +after 1582, vec on 3785 3804 24 26.4 37.8 2.7X +before 1582, vec off 10537 10582 39 9.5 105.4 1.0X +before 1582, vec on 4117 4146 25 24.3 41.2 2.5X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2812 2812 0 35.6 28.1 1.0X -before 1900, noop 2801 2801 0 35.7 28.0 1.0X -after 1900 18290 18290 0 5.5 182.9 0.2X -before 1900 22344 22344 0 4.5 223.4 0.1X +after 1900, noop 2853 2853 0 35.1 28.5 1.0X +before 1900, noop 2999 2999 0 33.3 30.0 1.0X +after 1900 16757 16757 0 6.0 167.6 0.2X +before 1900 21542 21542 0 4.6 215.4 0.1X OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 11257 11279 32 8.9 112.6 1.0X -after 1900, vec on 5296 5310 15 18.9 53.0 2.1X -before 1900, vec off 14700 14758 72 6.8 147.0 0.8X -before 1900, vec on 8576 8665 150 11.7 85.8 1.3X +after 1900, vec off 12212 12254 39 8.2 122.1 1.0X +after 1900, vec on 5369 5390 35 18.6 53.7 2.3X +before 1900, vec off 15661 15705 73 6.4 156.6 0.8X +before 1900, vec on 8720 8744 29 11.5 87.2 1.4X diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala index d6167f98b5a51..7caaa5376db7f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala @@ -25,8 +25,7 @@ import org.apache.spark.sql.DataFrame import org.apache.spark.sql.catalyst.util.DateTimeConstants.SECONDS_PER_DAY import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, LA} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._ -import org.apache.spark.sql.internal.SQLConf.ParquetOutputTimestampType +import org.apache.spark.sql.internal.SQLConf.{LegacyBehaviorPolicy, ParquetOutputTimestampType} object DateTime extends Enumeration { type DateTime = Value @@ -118,12 +117,12 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { private def caseName( modernDates: Boolean, dateTime: DateTime, - rebase: Option[Boolean] = None, + mode: Option[LegacyBehaviorPolicy.Value] = None, vec: Option[Boolean] = None): String = { val period = if (modernDates) "after" else "before" val year = if (dateTime == DATE) 1582 else 1900 val vecFlag = vec.map(flagToStr).map(flag => s", vec $flag").getOrElse("") - val rebaseFlag = rebase.map(flagToStr).map(flag => s", rebase $flag").getOrElse("") + val rebaseFlag = mode.map(_.toString).map(m => s", rebase $m").getOrElse("") s"$period $year$vecFlag$rebaseFlag" } @@ -131,10 +130,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { basePath: File, dateTime: DateTime, modernDates: Boolean, - rebase: Option[Boolean] = None): String = { + mode: Option[LegacyBehaviorPolicy.Value] = None): String = { val period = if (modernDates) "after" else "before" val year = if (dateTime == DATE) 1582 else 1900 - val rebaseFlag = rebase.map(flagToStr).map(flag => s"_$flag").getOrElse("") + val rebaseFlag = mode.map(_.toString).map(m => s"_$m").getOrElse("") basePath.getAbsolutePath + s"/${dateTime}_${period}_$year$rebaseFlag" } @@ -160,9 +159,10 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { output = output) benchmarkInputs(benchmark, rowsNum, dateTime) Seq(true, false).foreach { modernDates => - Seq(false, true).foreach { rebase => - benchmark.addCase(caseName(modernDates, dateTime, Some(rebase)), 1) { _ => - val mode = if (rebase) LEGACY else CORRECTED + LegacyBehaviorPolicy.values + .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION) + .foreach { mode => + benchmark.addCase(caseName(modernDates, dateTime, Some(mode)), 1) { _ => withSQLConf( SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime), SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) { @@ -170,7 +170,7 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { .write .mode("overwrite") .format("parquet") - .save(getPath(path, dateTime, modernDates, Some(rebase))) + .save(getPath(path, dateTime, modernDates, Some(mode))) } } } @@ -181,13 +181,15 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { s"Load $dateTime from parquet", rowsNum, output = output) Seq(true, false).foreach { modernDates => Seq(false, true).foreach { vec => - Seq(false, true).foreach { rebase => - val name = caseName(modernDates, dateTime, Some(rebase), Some(vec)) + LegacyBehaviorPolicy.values + .filterNot(v => !modernDates && v == LegacyBehaviorPolicy.EXCEPTION) + .foreach { mode => + val name = caseName(modernDates, dateTime, Some(mode), Some(vec)) benchmark2.addCase(name, 3) { _ => withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> vec.toString) { spark.read .format("parquet") - .load(getPath(path, dateTime, modernDates, Some(rebase))) + .load(getPath(path, dateTime, modernDates, Some(mode))) .noop() } }