diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt index 05896a4d69b4..74b19f2eef6a 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk11-results.txt @@ -2,153 +2,153 @@ Rebasing dates/timestamps in Parquet datasource ================================================================================================ -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 20023 20023 0 5.0 200.2 1.0X -before 1582, noop 10729 10729 0 9.3 107.3 1.9X -after 1582, rebase EXCEPTION 31834 31834 0 3.1 318.3 0.6X -after 1582, rebase LEGACY 31997 31997 0 3.1 320.0 0.6X -after 1582, rebase CORRECTED 31712 31712 0 3.2 317.1 0.6X -before 1582, rebase LEGACY 23663 23663 0 4.2 236.6 0.8X -before 1582, rebase CORRECTED 22749 22749 0 4.4 227.5 0.9X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1582, noop 21041 21041 0 4.8 210.4 1.0X +before 1582, noop 11202 11202 0 8.9 112.0 1.9X +after 1582, rebase EXCEPTION 32810 32810 0 3.0 328.1 0.6X +after 1582, rebase LEGACY 32530 32530 0 3.1 325.3 0.6X +after 1582, rebase CORRECTED 32849 32849 0 3.0 328.5 0.6X +before 1582, rebase LEGACY 23537 23537 0 4.2 235.4 0.9X +before 1582, rebase CORRECTED 22870 22870 0 4.4 228.7 0.9X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase EXCEPTION 12984 13262 257 7.7 129.8 1.0X -after 1582, vec off, rebase LEGACY 13278 13330 50 7.5 132.8 1.0X -after 1582, vec off, rebase CORRECTED 13202 13255 50 7.6 132.0 1.0X -after 1582, vec on, rebase EXCEPTION 3823 3853 40 26.2 38.2 3.4X -after 1582, vec on, rebase LEGACY 3846 3876 27 26.0 38.5 3.4X -after 1582, vec on, rebase CORRECTED 3775 3838 62 26.5 37.7 3.4X -before 1582, vec off, rebase LEGACY 13671 13692 26 7.3 136.7 0.9X -before 1582, vec off, rebase CORRECTED 13387 13476 106 7.5 133.9 1.0X -before 1582, vec on, rebase LEGACY 4477 4484 7 22.3 44.8 2.9X -before 1582, vec on, rebase CORRECTED 3729 3773 50 26.8 37.3 3.5X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1582, vec off, rebase EXCEPTION 13114 13225 104 7.6 131.1 1.0X +after 1582, vec off, rebase LEGACY 13175 13189 15 7.6 131.8 1.0X +after 1582, vec off, rebase CORRECTED 13080 13115 34 7.6 130.8 1.0X +after 1582, vec on, rebase EXCEPTION 3698 3726 29 27.0 37.0 3.5X +after 1582, vec on, rebase LEGACY 3730 3745 17 26.8 37.3 3.5X +after 1582, vec on, rebase CORRECTED 3714 3758 75 26.9 37.1 3.5X +before 1582, vec off, rebase LEGACY 13519 13575 63 7.4 135.2 1.0X +before 1582, vec off, rebase CORRECTED 13210 13309 108 7.6 132.1 1.0X +before 1582, vec on, rebase LEGACY 4459 4488 44 22.4 44.6 2.9X +before 1582, vec on, rebase CORRECTED 3661 3718 88 27.3 36.6 3.6X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 3020 3020 0 33.1 30.2 1.0X -before 1900, noop 3013 3013 0 33.2 30.1 1.0X -after 1900, rebase EXCEPTION 28796 28796 0 3.5 288.0 0.1X -after 1900, rebase LEGACY 28869 28869 0 3.5 288.7 0.1X -after 1900, rebase CORRECTED 28522 28522 0 3.5 285.2 0.1X -before 1900, rebase LEGACY 30594 30594 0 3.3 305.9 0.1X -before 1900, rebase CORRECTED 30743 30743 0 3.3 307.4 0.1X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1900, noop 2900 2900 0 34.5 29.0 1.0X +before 1900, noop 2848 2848 0 35.1 28.5 1.0X +after 1900, rebase EXCEPTION 27623 27623 0 3.6 276.2 0.1X +after 1900, rebase LEGACY 27305 27305 0 3.7 273.0 0.1X +after 1900, rebase CORRECTED 27715 27715 0 3.6 277.2 0.1X +before 1900, rebase LEGACY 30911 30911 0 3.2 309.1 0.1X +before 1900, rebase CORRECTED 27944 27944 0 3.6 279.4 0.1X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 19325 19468 135 5.2 193.3 1.0X -after 1900, vec off, rebase LEGACY 19568 19602 30 5.1 195.7 1.0X -after 1900, vec off, rebase CORRECTED 19532 19538 6 5.1 195.3 1.0X -after 1900, vec on, rebase EXCEPTION 9884 9990 94 10.1 98.8 2.0X -after 1900, vec on, rebase LEGACY 9933 9985 49 10.1 99.3 1.9X -after 1900, vec on, rebase CORRECTED 9967 10043 76 10.0 99.7 1.9X -before 1900, vec off, rebase LEGACY 24162 24198 37 4.1 241.6 0.8X -before 1900, vec off, rebase CORRECTED 24034 24056 20 4.2 240.3 0.8X -before 1900, vec on, rebase LEGACY 12548 12625 72 8.0 125.5 1.5X -before 1900, vec on, rebase CORRECTED 12580 12660 115 7.9 125.8 1.5X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1900, vec off, rebase EXCEPTION 16853 16885 41 5.9 168.5 1.0X +after 1900, vec off, rebase LEGACY 16804 16816 21 6.0 168.0 1.0X +after 1900, vec off, rebase CORRECTED 16985 17020 58 5.9 169.9 1.0X +after 1900, vec on, rebase EXCEPTION 7044 7063 19 14.2 70.4 2.4X +after 1900, vec on, rebase LEGACY 7183 7255 94 13.9 71.8 2.3X +after 1900, vec on, rebase CORRECTED 7047 7137 86 14.2 70.5 2.4X +before 1900, vec off, rebase LEGACY 20371 20458 81 4.9 203.7 0.8X +before 1900, vec off, rebase CORRECTED 17484 17541 54 5.7 174.8 1.0X +before 1900, vec on, rebase LEGACY 10284 10327 45 9.7 102.8 1.6X +before 1900, vec on, rebase CORRECTED 7044 7073 37 14.2 70.4 2.4X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 3159 3159 0 31.7 31.6 1.0X -before 1900, noop 3038 3038 0 32.9 30.4 1.0X -after 1900, rebase EXCEPTION 16885 16885 0 5.9 168.8 0.2X -after 1900, rebase LEGACY 17171 17171 0 5.8 171.7 0.2X -after 1900, rebase CORRECTED 17353 17353 0 5.8 173.5 0.2X -before 1900, rebase LEGACY 20579 20579 0 4.9 205.8 0.2X -before 1900, rebase CORRECTED 17544 17544 0 5.7 175.4 0.2X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1900, noop 2848 2848 0 35.1 28.5 1.0X +before 1900, noop 2855 2855 0 35.0 28.6 1.0X +after 1900, rebase EXCEPTION 15622 15622 0 6.4 156.2 0.2X +after 1900, rebase LEGACY 16148 16148 0 6.2 161.5 0.2X +after 1900, rebase CORRECTED 16946 16946 0 5.9 169.5 0.2X +before 1900, rebase LEGACY 19486 19486 0 5.1 194.9 0.1X +before 1900, rebase CORRECTED 17029 17029 0 5.9 170.3 0.2X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 16304 16345 58 6.1 163.0 1.0X -after 1900, vec off, rebase LEGACY 16503 16585 75 6.1 165.0 1.0X -after 1900, vec off, rebase CORRECTED 16413 16463 44 6.1 164.1 1.0X -after 1900, vec on, rebase EXCEPTION 5017 5034 29 19.9 50.2 3.2X -after 1900, vec on, rebase LEGACY 5060 5094 30 19.8 50.6 3.2X -after 1900, vec on, rebase CORRECTED 4969 4971 1 20.1 49.7 3.3X -before 1900, vec off, rebase LEGACY 19767 20001 203 5.1 197.7 0.8X -before 1900, vec off, rebase CORRECTED 16421 16465 38 6.1 164.2 1.0X -before 1900, vec on, rebase LEGACY 8535 8608 64 11.7 85.4 1.9X -before 1900, vec on, rebase CORRECTED 5044 5077 32 19.8 50.4 3.2X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1900, vec off, rebase EXCEPTION 15785 15848 56 6.3 157.9 1.0X +after 1900, vec off, rebase LEGACY 15935 15954 17 6.3 159.3 1.0X +after 1900, vec off, rebase CORRECTED 15976 16046 62 6.3 159.8 1.0X +after 1900, vec on, rebase EXCEPTION 4925 4941 20 20.3 49.3 3.2X +after 1900, vec on, rebase LEGACY 5033 5041 11 19.9 50.3 3.1X +after 1900, vec on, rebase CORRECTED 4946 4972 29 20.2 49.5 3.2X +before 1900, vec off, rebase LEGACY 18619 18782 176 5.4 186.2 0.8X +before 1900, vec off, rebase CORRECTED 15956 16018 56 6.3 159.6 1.0X +before 1900, vec on, rebase LEGACY 8461 8472 14 11.8 84.6 1.9X +before 1900, vec on, rebase CORRECTED 4953 4962 12 20.2 49.5 3.2X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2995 2995 0 33.4 29.9 1.0X -before 1900, noop 2981 2981 0 33.5 29.8 1.0X -after 1900, rebase EXCEPTION 16196 16196 0 6.2 162.0 0.2X -after 1900, rebase LEGACY 16550 16550 0 6.0 165.5 0.2X -after 1900, rebase CORRECTED 16908 16908 0 5.9 169.1 0.2X -before 1900, rebase LEGACY 20087 20087 0 5.0 200.9 0.1X -before 1900, rebase CORRECTED 17171 17171 0 5.8 171.7 0.2X - -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +after 1900, noop 3019 3019 0 33.1 30.2 1.0X +before 1900, noop 2896 2896 0 34.5 29.0 1.0X +after 1900, rebase EXCEPTION 15525 15525 0 6.4 155.2 0.2X +after 1900, rebase LEGACY 15903 15903 0 6.3 159.0 0.2X +after 1900, rebase CORRECTED 16468 16468 0 6.1 164.7 0.2X +before 1900, rebase LEGACY 19620 19620 0 5.1 196.2 0.2X +before 1900, rebase CORRECTED 16470 16470 0 6.1 164.7 0.2X + +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 16688 16787 88 6.0 166.9 1.0X -after 1900, vec off, rebase LEGACY 17383 17462 73 5.8 173.8 1.0X -after 1900, vec off, rebase CORRECTED 17317 17329 11 5.8 173.2 1.0X -after 1900, vec on, rebase EXCEPTION 6342 6348 6 15.8 63.4 2.6X -after 1900, vec on, rebase LEGACY 6500 6521 18 15.4 65.0 2.6X -after 1900, vec on, rebase CORRECTED 6164 6172 11 16.2 61.6 2.7X -before 1900, vec off, rebase LEGACY 20575 20665 81 4.9 205.7 0.8X -before 1900, vec off, rebase CORRECTED 17239 17290 61 5.8 172.4 1.0X -before 1900, vec on, rebase LEGACY 9310 9373 60 10.7 93.1 1.8X -before 1900, vec on, rebase CORRECTED 6091 6105 16 16.4 60.9 2.7X +after 1900, vec off, rebase EXCEPTION 16329 16357 26 6.1 163.3 1.0X +after 1900, vec off, rebase LEGACY 16609 16659 51 6.0 166.1 1.0X +after 1900, vec off, rebase CORRECTED 16659 16765 91 6.0 166.6 1.0X +after 1900, vec on, rebase EXCEPTION 6132 6162 28 16.3 61.3 2.7X +after 1900, vec on, rebase LEGACY 6344 6397 61 15.8 63.4 2.6X +after 1900, vec on, rebase CORRECTED 6023 6024 2 16.6 60.2 2.7X +before 1900, vec off, rebase LEGACY 19611 19626 13 5.1 196.1 0.8X +before 1900, vec off, rebase CORRECTED 16765 16784 19 6.0 167.7 1.0X +before 1900, vec on, rebase LEGACY 9136 9158 19 10.9 91.4 1.8X +before 1900, vec on, rebase CORRECTED 6023 6042 30 16.6 60.2 2.7X ================================================================================================ Rebasing dates/timestamps in ORC datasource ================================================================================================ -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 19583 19583 0 5.1 195.8 1.0X -before 1582, noop 10711 10711 0 9.3 107.1 1.8X -after 1582 27864 27864 0 3.6 278.6 0.7X -before 1582 19648 19648 0 5.1 196.5 1.0X +after 1582, noop 20934 20934 0 4.8 209.3 1.0X +before 1582, noop 11098 11098 0 9.0 111.0 1.9X +after 1582 29249 29249 0 3.4 292.5 0.7X +before 1582 20059 20059 0 5.0 200.6 1.0X -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 10383 10560 192 9.6 103.8 1.0X -after 1582, vec on 3844 3864 33 26.0 38.4 2.7X -before 1582, vec off 10867 10916 48 9.2 108.7 1.0X -before 1582, vec on 4158 4170 12 24.0 41.6 2.5X +after 1582, vec off 10751 10802 56 9.3 107.5 1.0X +after 1582, vec on 3815 3870 62 26.2 38.1 2.8X +before 1582, vec off 11144 11174 37 9.0 111.4 1.0X +before 1582, vec on 4120 4126 8 24.3 41.2 2.6X -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2989 2989 0 33.5 29.9 1.0X -before 1900, noop 3000 3000 0 33.3 30.0 1.0X -after 1900 19426 19426 0 5.1 194.3 0.2X -before 1900 23282 23282 0 4.3 232.8 0.1X +after 1900, noop 2858 2858 0 35.0 28.6 1.0X +before 1900, noop 2859 2859 0 35.0 28.6 1.0X +after 1900 17098 17098 0 5.8 171.0 0.2X +before 1900 20639 20639 0 4.8 206.4 0.1X -OpenJDK 64-Bit Server VM 11.0.7+10-post-Ubuntu-2ubuntu218.04 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 11.0.8+10-post-Ubuntu-0ubuntu118.04.1 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 12089 12102 15 8.3 120.9 1.0X -after 1900, vec on 5210 5325 100 19.2 52.1 2.3X -before 1900, vec off 15320 15373 46 6.5 153.2 0.8X -before 1900, vec on 7937 7970 48 12.6 79.4 1.5X +after 1900, vec off 12292 12318 23 8.1 122.9 1.0X +after 1900, vec on 5198 5271 95 19.2 52.0 2.4X +before 1900, vec off 15108 15145 53 6.6 151.1 0.8X +before 1900, vec on 8085 8277 245 12.4 80.8 1.5X diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt index 3e94d6c6fcfa..07b156a62e2e 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt @@ -2,153 +2,153 @@ Rebasing dates/timestamps in Parquet datasource ================================================================================================ -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 23300 23300 0 4.3 233.0 1.0X -before 1582, noop 10585 10585 0 9.4 105.9 2.2X -after 1582, rebase EXCEPTION 35215 35215 0 2.8 352.1 0.7X -after 1582, rebase LEGACY 34927 34927 0 2.9 349.3 0.7X -after 1582, rebase CORRECTED 35479 35479 0 2.8 354.8 0.7X -before 1582, rebase LEGACY 22767 22767 0 4.4 227.7 1.0X -before 1582, rebase CORRECTED 22527 22527 0 4.4 225.3 1.0X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1582, noop 22736 22736 0 4.4 227.4 1.0X +before 1582, noop 10512 10512 0 9.5 105.1 2.2X +after 1582, rebase EXCEPTION 35759 35759 0 2.8 357.6 0.6X +after 1582, rebase LEGACY 36229 36229 0 2.8 362.3 0.6X +after 1582, rebase CORRECTED 35489 35489 0 2.8 354.9 0.6X +before 1582, rebase LEGACY 23514 23514 0 4.3 235.1 1.0X +before 1582, rebase CORRECTED 23234 23234 0 4.3 232.3 1.0X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase EXCEPTION 13480 13577 94 7.4 134.8 1.0X -after 1582, vec off, rebase LEGACY 13466 13586 118 7.4 134.7 1.0X -after 1582, vec off, rebase CORRECTED 13526 13558 41 7.4 135.3 1.0X -after 1582, vec on, rebase EXCEPTION 3759 3778 28 26.6 37.6 3.6X -after 1582, vec on, rebase LEGACY 3957 4004 57 25.3 39.6 3.4X -after 1582, vec on, rebase CORRECTED 3739 3755 25 26.7 37.4 3.6X -before 1582, vec off, rebase LEGACY 13986 14038 67 7.1 139.9 1.0X -before 1582, vec off, rebase CORRECTED 13453 13491 49 7.4 134.5 1.0X -before 1582, vec on, rebase LEGACY 4716 4724 10 21.2 47.2 2.9X -before 1582, vec on, rebase CORRECTED 3701 3750 50 27.0 37.0 3.6X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1582, vec off, rebase EXCEPTION 13036 13121 85 7.7 130.4 1.0X +after 1582, vec off, rebase LEGACY 13567 13631 55 7.4 135.7 1.0X +after 1582, vec off, rebase CORRECTED 13476 13498 28 7.4 134.8 1.0X +after 1582, vec on, rebase EXCEPTION 3676 3679 3 27.2 36.8 3.5X +after 1582, vec on, rebase LEGACY 3842 3863 19 26.0 38.4 3.4X +after 1582, vec on, rebase CORRECTED 3706 3756 69 27.0 37.1 3.5X +before 1582, vec off, rebase LEGACY 13781 13832 68 7.3 137.8 0.9X +before 1582, vec off, rebase CORRECTED 13414 13445 28 7.5 134.1 1.0X +before 1582, vec on, rebase LEGACY 4774 4788 14 20.9 47.7 2.7X +before 1582, vec on, rebase CORRECTED 3650 3691 38 27.4 36.5 3.6X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2790 2790 0 35.8 27.9 1.0X -before 1900, noop 2812 2812 0 35.6 28.1 1.0X -after 1900, rebase EXCEPTION 24789 24789 0 4.0 247.9 0.1X -after 1900, rebase LEGACY 24539 24539 0 4.1 245.4 0.1X -after 1900, rebase CORRECTED 24543 24543 0 4.1 245.4 0.1X -before 1900, rebase LEGACY 30496 30496 0 3.3 305.0 0.1X -before 1900, rebase CORRECTED 30428 30428 0 3.3 304.3 0.1X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1900, noop 2696 2696 0 37.1 27.0 1.0X +before 1900, noop 2687 2687 0 37.2 26.9 1.0X +after 1900, rebase EXCEPTION 29085 29085 0 3.4 290.9 0.1X +after 1900, rebase LEGACY 29789 29789 0 3.4 297.9 0.1X +after 1900, rebase CORRECTED 29563 29563 0 3.4 295.6 0.1X +before 1900, rebase LEGACY 34033 34033 0 2.9 340.3 0.1X +before 1900, rebase CORRECTED 29687 29687 0 3.4 296.9 0.1X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 17106 17192 75 5.8 171.1 1.0X -after 1900, vec off, rebase LEGACY 17273 17337 55 5.8 172.7 1.0X -after 1900, vec off, rebase CORRECTED 17073 17215 128 5.9 170.7 1.0X -after 1900, vec on, rebase EXCEPTION 8903 8976 117 11.2 89.0 1.9X -after 1900, vec on, rebase LEGACY 8793 8876 84 11.4 87.9 1.9X -after 1900, vec on, rebase CORRECTED 8820 8878 53 11.3 88.2 1.9X -before 1900, vec off, rebase LEGACY 20997 21069 82 4.8 210.0 0.8X -before 1900, vec off, rebase CORRECTED 20874 20946 90 4.8 208.7 0.8X -before 1900, vec on, rebase LEGACY 12024 12090 58 8.3 120.2 1.4X -before 1900, vec on, rebase CORRECTED 12020 12069 64 8.3 120.2 1.4X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1900, vec off, rebase EXCEPTION 16623 16711 78 6.0 166.2 1.0X +after 1900, vec off, rebase LEGACY 16525 16641 103 6.1 165.3 1.0X +after 1900, vec off, rebase CORRECTED 16698 16847 133 6.0 167.0 1.0X +after 1900, vec on, rebase EXCEPTION 8614 8723 97 11.6 86.1 1.9X +after 1900, vec on, rebase LEGACY 9790 9812 20 10.2 97.9 1.7X +after 1900, vec on, rebase CORRECTED 8607 8671 73 11.6 86.1 1.9X +before 1900, vec off, rebase LEGACY 21389 21553 142 4.7 213.9 0.8X +before 1900, vec off, rebase CORRECTED 17539 17545 6 5.7 175.4 0.9X +before 1900, vec on, rebase LEGACY 13594 13627 40 7.4 135.9 1.2X +before 1900, vec on, rebase CORRECTED 8620 8666 73 11.6 86.2 1.9X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2939 2939 0 34.0 29.4 1.0X -before 1900, noop 2917 2917 0 34.3 29.2 1.0X -after 1900, rebase EXCEPTION 15954 15954 0 6.3 159.5 0.2X -after 1900, rebase LEGACY 16402 16402 0 6.1 164.0 0.2X -after 1900, rebase CORRECTED 16541 16541 0 6.0 165.4 0.2X -before 1900, rebase LEGACY 20500 20500 0 4.9 205.0 0.1X -before 1900, rebase CORRECTED 16764 16764 0 6.0 167.6 0.2X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1900, noop 2755 2755 0 36.3 27.5 1.0X +before 1900, noop 2819 2819 0 35.5 28.2 1.0X +after 1900, rebase EXCEPTION 16742 16742 0 6.0 167.4 0.2X +after 1900, rebase LEGACY 16978 16978 0 5.9 169.8 0.2X +after 1900, rebase CORRECTED 17508 17508 0 5.7 175.1 0.2X +before 1900, rebase LEGACY 21961 21961 0 4.6 219.6 0.1X +before 1900, rebase CORRECTED 17770 17770 0 5.6 177.7 0.2X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 15607 15655 81 6.4 156.1 1.0X -after 1900, vec off, rebase LEGACY 15616 15676 54 6.4 156.2 1.0X -after 1900, vec off, rebase CORRECTED 15634 15732 108 6.4 156.3 1.0X -after 1900, vec on, rebase EXCEPTION 5041 5057 16 19.8 50.4 3.1X -after 1900, vec on, rebase LEGACY 5516 5539 29 18.1 55.2 2.8X -after 1900, vec on, rebase CORRECTED 5087 5104 28 19.7 50.9 3.1X -before 1900, vec off, rebase LEGACY 19262 19338 79 5.2 192.6 0.8X -before 1900, vec off, rebase CORRECTED 15718 15755 53 6.4 157.2 1.0X -before 1900, vec on, rebase LEGACY 10147 10240 114 9.9 101.5 1.5X -before 1900, vec on, rebase CORRECTED 5062 5080 21 19.8 50.6 3.1X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1900, vec off, rebase EXCEPTION 15311 15405 82 6.5 153.1 1.0X +after 1900, vec off, rebase LEGACY 15501 15578 73 6.5 155.0 1.0X +after 1900, vec off, rebase CORRECTED 15331 15472 123 6.5 153.3 1.0X +after 1900, vec on, rebase EXCEPTION 4976 5008 38 20.1 49.8 3.1X +after 1900, vec on, rebase LEGACY 5366 5443 67 18.6 53.7 2.9X +after 1900, vec on, rebase CORRECTED 4977 4982 9 20.1 49.8 3.1X +before 1900, vec off, rebase LEGACY 19205 19281 65 5.2 192.1 0.8X +before 1900, vec off, rebase CORRECTED 15458 15490 28 6.5 154.6 1.0X +before 1900, vec on, rebase LEGACY 9878 9933 79 10.1 98.8 1.5X +before 1900, vec on, rebase CORRECTED 4886 4961 66 20.5 48.9 3.1X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2915 2915 0 34.3 29.2 1.0X -before 1900, noop 2894 2894 0 34.6 28.9 1.0X -after 1900, rebase EXCEPTION 15545 15545 0 6.4 155.4 0.2X -after 1900, rebase LEGACY 15840 15840 0 6.3 158.4 0.2X -after 1900, rebase CORRECTED 16324 16324 0 6.1 163.2 0.2X -before 1900, rebase LEGACY 20359 20359 0 4.9 203.6 0.1X -before 1900, rebase CORRECTED 16292 16292 0 6.1 162.9 0.2X - -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +after 1900, noop 2836 2836 0 35.3 28.4 1.0X +before 1900, noop 2813 2813 0 35.6 28.1 1.0X +after 1900, rebase EXCEPTION 16549 16549 0 6.0 165.5 0.2X +after 1900, rebase LEGACY 16296 16296 0 6.1 163.0 0.2X +after 1900, rebase CORRECTED 16913 16913 0 5.9 169.1 0.2X +before 1900, rebase LEGACY 21150 21150 0 4.7 211.5 0.1X +before 1900, rebase CORRECTED 17090 17090 0 5.9 170.9 0.2X + +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 15857 16015 223 6.3 158.6 1.0X -after 1900, vec off, rebase LEGACY 16174 16231 63 6.2 161.7 1.0X -after 1900, vec off, rebase CORRECTED 16353 16400 67 6.1 163.5 1.0X -after 1900, vec on, rebase EXCEPTION 6449 6459 9 15.5 64.5 2.5X -after 1900, vec on, rebase LEGACY 7028 7035 6 14.2 70.3 2.3X -after 1900, vec on, rebase CORRECTED 6585 6623 37 15.2 65.8 2.4X -before 1900, vec off, rebase LEGACY 19929 20027 95 5.0 199.3 0.8X -before 1900, vec off, rebase CORRECTED 16401 16451 49 6.1 164.0 1.0X -before 1900, vec on, rebase LEGACY 10517 10563 40 9.5 105.2 1.5X -before 1900, vec on, rebase CORRECTED 6659 6675 26 15.0 66.6 2.4X +after 1900, vec off, rebase EXCEPTION 15706 15823 132 6.4 157.1 1.0X +after 1900, vec off, rebase LEGACY 16100 16194 88 6.2 161.0 1.0X +after 1900, vec off, rebase CORRECTED 16227 16282 81 6.2 162.3 1.0X +after 1900, vec on, rebase EXCEPTION 6383 6404 26 15.7 63.8 2.5X +after 1900, vec on, rebase LEGACY 6994 7006 15 14.3 69.9 2.2X +after 1900, vec on, rebase CORRECTED 6580 6597 15 15.2 65.8 2.4X +before 1900, vec off, rebase LEGACY 19601 19674 82 5.1 196.0 0.8X +before 1900, vec off, rebase CORRECTED 16188 16215 25 6.2 161.9 1.0X +before 1900, vec on, rebase LEGACY 10305 10360 51 9.7 103.1 1.5X +before 1900, vec on, rebase CORRECTED 6573 6600 28 15.2 65.7 2.4X ================================================================================================ Rebasing dates/timestamps in ORC datasource ================================================================================================ -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 22782 22782 0 4.4 227.8 1.0X -before 1582, noop 10555 10555 0 9.5 105.6 2.2X -after 1582 31497 31497 0 3.2 315.0 0.7X -before 1582 19803 19803 0 5.0 198.0 1.2X +after 1582, noop 22766 22766 0 4.4 227.7 1.0X +before 1582, noop 10535 10535 0 9.5 105.3 2.2X +after 1582 31037 31037 0 3.2 310.4 0.7X +before 1582 19755 19755 0 5.1 197.6 1.2X -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 10180 10214 44 9.8 101.8 1.0X -after 1582, vec on 3785 3804 24 26.4 37.8 2.7X -before 1582, vec off 10537 10582 39 9.5 105.4 1.0X -before 1582, vec on 4117 4146 25 24.3 41.2 2.5X +after 1582, vec off 11137 11165 37 9.0 111.4 1.0X +after 1582, vec on 3701 3734 51 27.0 37.0 3.0X +before 1582, vec off 11379 11409 50 8.8 113.8 1.0X +before 1582, vec on 4110 4160 57 24.3 41.1 2.7X -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2853 2853 0 35.1 28.5 1.0X -before 1900, noop 2999 2999 0 33.3 30.0 1.0X -after 1900 16757 16757 0 6.0 167.6 0.2X -before 1900 21542 21542 0 4.6 215.4 0.1X +after 1900, noop 2830 2830 0 35.3 28.3 1.0X +before 1900, noop 2867 2867 0 34.9 28.7 1.0X +after 1900 17867 17867 0 5.6 178.7 0.2X +before 1900 21555 21555 0 4.6 215.6 0.1X -OpenJDK 64-Bit Server VM 1.8.0_252-8u252-b09-1~18.04-b09 on Linux 4.15.0-1063-aws +OpenJDK 64-Bit Server VM 1.8.0_265-8u265-b01-0ubuntu2~18.04-b01 on Linux 5.3.0-1034-aws Intel(R) Xeon(R) CPU E5-2670 v2 @ 2.50GHz Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 12212 12254 39 8.2 122.1 1.0X -after 1900, vec on 5369 5390 35 18.6 53.7 2.3X -before 1900, vec off 15661 15705 73 6.4 156.6 0.8X -before 1900, vec on 8720 8744 29 11.5 87.2 1.4X +after 1900, vec off 12245 12269 24 8.2 122.5 1.0X +after 1900, vec on 5258 5303 63 19.0 52.6 2.3X +before 1900, vec off 15698 15777 119 6.4 157.0 0.8X +before 1900, vec on 8568 8674 138 11.7 85.7 1.4X diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala index 7caaa5376db7..bc94d1f23580 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/DateTimeRebaseBenchmark.scala @@ -165,7 +165,8 @@ object DateTimeRebaseBenchmark extends SqlBasedBenchmark { benchmark.addCase(caseName(modernDates, dateTime, Some(mode)), 1) { _ => withSQLConf( SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key -> getOutputType(dateTime), - SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString) { + SQLConf.LEGACY_PARQUET_REBASE_MODE_IN_WRITE.key -> mode.toString, + SQLConf.LEGACY_PARQUET_INT96_REBASE_MODE_IN_WRITE.key -> mode.toString) { genDF(rowsNum, dateTime, modernDates) .write .mode("overwrite")