diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 52adec20e5c1..f73267a95fa3 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -54,7 +54,7 @@ jobs: steps: - name: Generate matrix id: set-matrix - run: echo "::set-output name=matrix::["`seq -s, 1 $SPARK_BENCHMARK_NUM_SPLITS`"]" + run: echo "matrix=["`seq -s, 1 $SPARK_BENCHMARK_NUM_SPLITS`"]" >> $GITHUB_OUTPUT # Any TPC-DS related updates on this job need to be applied to tpcds-1g job of build_and_test.yml as well tpcds-1g-gen: @@ -65,7 +65,7 @@ jobs: SPARK_LOCAL_IP: localhost steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to get diff files with: fetch-depth: 0 @@ -95,7 +95,7 @@ jobs: key: tpcds-${{ hashFiles('.github/workflows/benchmark.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }} - name: Checkout tpcds-kit repository if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: databricks/tpcds-kit ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069 @@ -133,7 +133,7 @@ jobs: SPARK_TPCDS_DATA: ${{ github.workspace }}/tpcds-sf-1 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to get diff files with: fetch-depth: 0 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 64dbe30012cd..8411170e2d5c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -63,7 +63,7 @@ jobs: }} steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -103,16 +103,15 @@ jobs: \"k8s-integration-tests\" : \"true\", }" echo $precondition # For debugging - # GitHub Actions set-output doesn't take newlines - # https://github.community/t/set-output-truncates-multiline-strings/16852/3 - precondition="${precondition//$'\n'/'%0A'}" - echo "::set-output name=required::$precondition" + # Remove `\n` to avoid "Invalid format" error + precondition="${precondition//$'\n'/}}" + echo "required=$precondition" >> $GITHUB_OUTPUT else # This is usually set by scheduled jobs. precondition='${{ inputs.jobs }}' echo $precondition # For debugging - precondition="${precondition//$'\n'/'%0A'}" - echo "::set-output name=required::$precondition" + precondition="${precondition//$'\n'/}" + echo "required=$precondition" >> $GITHUB_OUTPUT fi - name: Generate infra image URL id: infra-image-outputs @@ -121,7 +120,7 @@ jobs: REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') IMG_NAME="apache-spark-ci-image:${{ inputs.branch }}-${{ github.run_id }}" IMG_URL="ghcr.io/$REPO_OWNER/$IMG_NAME" - echo ::set-output name=image_url::$IMG_URL + echo "image_url=$IMG_URL" >> $GITHUB_OUTPUT # Build: build Spark and run the tests for specified modules. build: @@ -195,7 +194,7 @@ jobs: SPARK_LOCAL_IP: localhost steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -243,7 +242,7 @@ jobs: - name: Install Python packages (Python 3.8) if: (contains(matrix.modules, 'sql') && !contains(matrix.modules, 'sql-')) run: | - python3.8 -m pip install 'numpy>=1.20.0' pyarrow pandas scipy unittest-xml-reporting grpcio protobuf + python3.8 -m pip install 'numpy>=1.20.0' pyarrow pandas scipy unittest-xml-reporting 'grpcio==1.48.1' 'protobuf==4.21.6' python3.8 -m pip list # Run the tests. - name: Run tests @@ -286,7 +285,7 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -300,12 +299,12 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: ./dev/infra/ push: true @@ -349,7 +348,7 @@ jobs: METASPACE_SIZE: 1g steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -438,7 +437,7 @@ jobs: SKIP_MIMA: true steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # In order to fetch changed files with: fetch-depth: 0 @@ -508,7 +507,7 @@ jobs: image: ${{ needs.precondition.outputs.image_url }} steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -587,7 +586,7 @@ jobs: # See also https://issues.apache.org/jira/browse/SPARK-38279. python3.9 -m pip install 'sphinx<3.1.0' mkdocs pydata_sphinx_theme ipython nbsphinx numpydoc 'jinja2<3.0.0' 'markupsafe==2.0.1' 'pyzmq<24.0.0' python3.9 -m pip install ipython_genutils # See SPARK-38517 - python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' grpcio protobuf mypy-protobuf + python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'grpcio==1.48.1' 'protobuf==4.21.6' 'mypy-protobuf==3.3.0' python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421 apt-get update -y apt-get install -y ruby ruby-dev @@ -635,7 +634,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -684,7 +683,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -732,7 +731,7 @@ jobs: SPARK_LOCAL_IP: localhost steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -773,7 +772,7 @@ jobs: key: tpcds-${{ hashFiles('.github/workflows/build_and_test.yml', 'sql/core/src/test/scala/org/apache/spark/sql/TPCDSSchema.scala') }} - name: Checkout tpcds-kit repository if: steps.cache-tpcds-sf-1.outputs.cache-hit != 'true' - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: databricks/tpcds-kit ref: 2a5078a782192ddb6efbcead8de9973d6ab4f069 @@ -834,7 +833,7 @@ jobs: SKIP_MIMA: true steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark @@ -891,7 +890,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 repository: apache/spark diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index b568b84c5b3c..b8aae945599d 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -38,20 +38,20 @@ jobs: packages: write steps: - name: Checkout Spark repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: ./dev/infra/ push: true diff --git a/.github/workflows/notify_test_workflow.yml b/.github/workflows/notify_test_workflow.yml index c9b5c54f3622..6fb776d70834 100644 --- a/.github/workflows/notify_test_workflow.yml +++ b/.github/workflows/notify_test_workflow.yml @@ -36,7 +36,7 @@ jobs: checks: write steps: - name: "Notify test workflow" - uses: actions/github-script@f05a81df23035049204b043b50c3322045ce7eb3 # pin@v3 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | @@ -80,7 +80,7 @@ jobs: status = 'completed' const conclusion = 'action_required' - github.checks.create({ + github.rest.checks.create({ owner: context.repo.owner, repo: context.repo.repo, name: name, @@ -132,7 +132,7 @@ jobs: + '/actions/runs/' + run_id - github.checks.create({ + github.rest.checks.create({ owner: context.repo.owner, repo: context.repo.repo, name: name, diff --git a/.github/workflows/publish_snapshot.yml b/.github/workflows/publish_snapshot.yml index d375a4149ded..a8251aa5b67c 100644 --- a/.github/workflows/publish_snapshot.yml +++ b/.github/workflows/publish_snapshot.yml @@ -37,7 +37,7 @@ jobs: - branch-3.1 steps: - name: Checkout Spark repository - uses: actions/checkout@61b9e3751b92087fd0b06925ba6dd6314e06f089 # pin@master + uses: actions/checkout@v3 with: ref: ${{ matrix.branch }} - name: Cache Maven local repository diff --git a/.github/workflows/update_build_status.yml b/.github/workflows/update_build_status.yml index 7f3826817dfb..05cf4914a25c 100644 --- a/.github/workflows/update_build_status.yml +++ b/.github/workflows/update_build_status.yml @@ -32,7 +32,7 @@ jobs: checks: write steps: - name: "Update build status" - uses: actions/github-script@f05a81df23035049204b043b50c3322045ce7eb3 # pin@v3 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/connector/connect/src/main/protobuf/spark/connect/base.proto b/connector/connect/src/main/protobuf/spark/connect/base.proto index 450f60d6aa5d..390a8b156dc4 100644 --- a/connector/connect/src/main/protobuf/spark/connect/base.proto +++ b/connector/connect/src/main/protobuf/spark/connect/base.proto @@ -62,7 +62,7 @@ message Response { // Result type oneof result_type { ArrowBatch batch = 2; - CSVBatch csv_batch = 3; + JSONBatch json_batch = 3; } // Metrics for the query execution. Typically, this field is only present in the last @@ -78,9 +78,12 @@ message Response { bytes schema = 5; } - message CSVBatch { + // Message type when the result is returned as JSON. This is essentially a bulk wrapper + // for the JSON result of a Spark DataFrame. All rows are returned in the JSON record format + // of `{col -> row}`. + message JSONBatch { int64 row_count = 1; - string data = 2; + bytes data = 2; } message Metrics { diff --git a/connector/connect/src/main/protobuf/spark/connect/relations.proto b/connector/connect/src/main/protobuf/spark/connect/relations.proto index 353fbebd0460..7dbde775ee88 100644 --- a/connector/connect/src/main/protobuf/spark/connect/relations.proto +++ b/connector/connect/src/main/protobuf/spark/connect/relations.proto @@ -43,6 +43,7 @@ message Relation { LocalRelation local_relation = 11; Sample sample = 12; Offset offset = 13; + Deduplicate deduplicate = 14; Unknown unknown = 999; } @@ -67,11 +68,21 @@ message SQL { message Read { oneof read_type { NamedTable named_table = 1; + DataSource data_source = 2; } message NamedTable { string unparsed_identifier = 1; } + + message DataSource { + // Required. Supported formats include: parquet, orc, text, json, parquet, csv, avro. + string format = 1; + // Optional. If not set, Spark will infer the schema. + string schema = 2; + // The key is case insensitive. + map options = 3; + } } // Projection of a bag of expressions for a given input relation. @@ -171,6 +182,14 @@ message Sort { } } +// Relation of type [[Deduplicate]] which have duplicate rows removed, could consider either only +// the subset of columns or all the columns. +message Deduplicate { + Relation input = 1; + repeated string column_names = 2; + bool all_columns_as_keys = 3; +} + message LocalRelation { repeated Expression.QualifiedAttribute attributes = 1; // TODO: support local data. @@ -182,5 +201,9 @@ message Sample { double lower_bound = 2; double upper_bound = 3; bool with_replacement = 4; - int64 seed = 5; + Seed seed = 5; + + message Seed { + int64 seed = 1; + } } diff --git a/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala b/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala index 8a267dff7d78..4630c86049c5 100644 --- a/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala +++ b/connector/connect/src/main/scala/org/apache/spark/sql/connect/dsl/package.scala @@ -215,6 +215,26 @@ package object dsl { .build() } + def deduplicate(colNames: Seq[String]): proto.Relation = + proto.Relation + .newBuilder() + .setDeduplicate( + proto.Deduplicate + .newBuilder() + .setInput(logicalPlan) + .addAllColumnNames(colNames.asJava)) + .build() + + def distinct(): proto.Relation = + proto.Relation + .newBuilder() + .setDeduplicate( + proto.Deduplicate + .newBuilder() + .setInput(logicalPlan) + .setAllColumnsAsKeys(true)) + .build() + def join( otherPlan: proto.Relation, joinType: JoinType = JoinType.JOIN_TYPE_INNER, @@ -252,7 +272,8 @@ package object dsl { .setUpperBound(upperBound) .setLowerBound(lowerBound) .setWithReplacement(withReplacement) - .setSeed(seed)) + .setSeed(proto.Sample.Seed.newBuilder().setSeed(seed).build()) + .build()) .build() } diff --git a/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 6a6b5a15a087..880618cc3338 100644 --- a/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/connector/connect/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -27,8 +27,11 @@ import org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Expression} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.catalyst.plans.{logical, FullOuter, Inner, JoinType, LeftAnti, LeftOuter, LeftSemi, RightOuter} -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Sample, SubqueryAlias} +import org.apache.spark.sql.catalyst.plans.logical.{Deduplicate, LogicalPlan, Sample, SubqueryAlias} +import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap +import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.types._ +import org.apache.spark.util.Utils final case class InvalidPlanInput( private val message: String = "", @@ -59,6 +62,7 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { case proto.Relation.RelTypeCase.OFFSET => transformOffset(rel.getOffset) case proto.Relation.RelTypeCase.JOIN => transformJoin(rel.getJoin) case proto.Relation.RelTypeCase.UNION => transformUnion(rel.getUnion) + case proto.Relation.RelTypeCase.DEDUPLICATE => transformDeduplicate(rel.getDeduplicate) case proto.Relation.RelTypeCase.SORT => transformSort(rel.getSort) case proto.Relation.RelTypeCase.AGGREGATE => transformAggregate(rel.getAggregate) case proto.Relation.RelTypeCase.SQL => transformSql(rel.getSql) @@ -77,7 +81,7 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { /** * All fields of [[proto.Sample]] are optional. However, given those are proto primitive types, - * we cannot differentiate if the fied is not or set when the field's value equals to the type + * we cannot differentiate if the field is not or set when the field's value equals to the type * default value. In the future if this ever become a problem, one solution could be that to * wrap such fields into proto messages. */ @@ -86,10 +90,41 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { rel.getLowerBound, rel.getUpperBound, rel.getWithReplacement, - rel.getSeed, + if (rel.hasSeed) rel.getSeed.getSeed else Utils.random.nextLong, transformRelation(rel.getInput)) } + private def transformDeduplicate(rel: proto.Deduplicate): LogicalPlan = { + if (!rel.hasInput) { + throw InvalidPlanInput("Deduplicate needs a plan input") + } + if (rel.getAllColumnsAsKeys && rel.getColumnNamesCount > 0) { + throw InvalidPlanInput("Cannot deduplicate on both all columns and a subset of columns") + } + if (!rel.getAllColumnsAsKeys && rel.getColumnNamesCount == 0) { + throw InvalidPlanInput( + "Deduplicate requires to either deduplicate on all columns or a subset of columns") + } + val queryExecution = new QueryExecution(session, transformRelation(rel.getInput)) + val resolver = session.sessionState.analyzer.resolver + val allColumns = queryExecution.analyzed.output + if (rel.getAllColumnsAsKeys) { + Deduplicate(allColumns, queryExecution.analyzed) + } else { + val toGroupColumnNames = rel.getColumnNamesList.asScala.toSeq + val groupCols = toGroupColumnNames.flatMap { (colName: String) => + // It is possibly there are more than one columns with the same name, + // so we call filter instead of find. + val cols = allColumns.filter(col => resolver(col.name, colName)) + if (cols.isEmpty) { + throw InvalidPlanInput(s"Invalid deduplicate column ${colName}") + } + cols + } + Deduplicate(groupCols, queryExecution.analyzed) + } + } + private def transformLocalRelation(rel: proto.LocalRelation): LogicalPlan = { val attributes = rel.getAttributesList.asScala.map(transformAttribute(_)).toSeq new org.apache.spark.sql.catalyst.plans.logical.LocalRelation(attributes) @@ -112,7 +147,19 @@ class SparkConnectPlanner(plan: proto.Relation, session: SparkSession) { } else { child } - case _ => throw InvalidPlanInput() + case proto.Read.ReadTypeCase.DATA_SOURCE => + if (rel.getDataSource.getFormat == "") { + throw InvalidPlanInput("DataSource requires a format") + } + val localMap = CaseInsensitiveMap[String](rel.getDataSource.getOptionsMap.asScala.toMap) + val reader = session.read + reader.format(rel.getDataSource.getFormat) + localMap.foreach { case (key, value) => reader.option(key, value) } + if (rel.getDataSource.getSchema != null) { + reader.schema(rel.getDataSource.getSchema) + } + reader.load().queryExecution.analyzed + case _ => throw InvalidPlanInput("Does not support " + rel.getReadTypeCase.name()) } baseRelation } diff --git a/connector/connect/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectStreamHandler.scala b/connector/connect/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectStreamHandler.scala index 84a6efb2baab..997b0f6b6d82 100644 --- a/connector/connect/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectStreamHandler.scala +++ b/connector/connect/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectStreamHandler.scala @@ -22,6 +22,7 @@ import scala.collection.JavaConverters._ import com.google.protobuf.ByteString import io.grpc.stub.StreamObserver +import org.apache.spark.SparkException import org.apache.spark.annotation.{Since, Unstable} import org.apache.spark.connect.proto import org.apache.spark.connect.proto.{Request, Response} @@ -32,12 +33,14 @@ import org.apache.spark.sql.connect.planner.SparkConnectPlanner import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, QueryStageExec} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.util.ArrowUtils @Unstable @Since("3.4.0") class SparkConnectStreamHandler(responseObserver: StreamObserver[Response]) extends Logging { + // The maximum batch size in bytes for a single batch of data to be returned via proto. + val MAX_BATCH_SIZE: Long = 10 * 1024 * 1024 + def handle(v: Request): Unit = { val session = SparkConnectService.getOrCreateIsolatedSession(v.getUserContext.getUserId).session @@ -57,21 +60,66 @@ class SparkConnectStreamHandler(responseObserver: StreamObserver[Response]) exte processRows(request.getClientId, rows) } - private def processRows(clientId: String, rows: DataFrame) = { + private[connect] def processRows(clientId: String, rows: DataFrame): Unit = { val timeZoneId = SQLConf.get.sessionLocalTimeZone - val schema = - ByteString.copyFrom(ArrowUtils.toArrowSchema(rows.schema, timeZoneId).toByteArray) - - val textSchema = rows.schema.fields.map(f => f.name).mkString("|") - val data = rows.collect().map(x => x.toSeq.mkString("|")).mkString("\n") - val bbb = proto.Response.CSVBatch.newBuilder - .setRowCount(-1) - .setData(textSchema ++ "\n" ++ data) - .build() - val response = proto.Response.newBuilder().setClientId(clientId).setCsvBatch(bbb).build() - // Send all the data - responseObserver.onNext(response) + // Only process up to 10MB of data. + val sb = new StringBuilder + var rowCount = 0 + rows.toJSON + .collect() + .foreach(row => { + + // There are a few cases to cover here. + // 1. The aggregated buffer size is larger than the MAX_BATCH_SIZE + // -> send the current batch and reset. + // 2. The aggregated buffer size is smaller than the MAX_BATCH_SIZE + // -> append the row to the buffer. + // 3. The row in question is larger than the MAX_BATCH_SIZE + // -> fail the query. + + // Case 3. - Fail + if (row.size > MAX_BATCH_SIZE) { + throw SparkException.internalError( + s"Serialized row is larger than MAX_BATCH_SIZE: ${row.size} > ${MAX_BATCH_SIZE}") + } + + // Case 1 - FLush and send. + if (sb.size + row.size > MAX_BATCH_SIZE) { + val response = proto.Response.newBuilder().setClientId(clientId) + val batch = proto.Response.JSONBatch + .newBuilder() + .setData(ByteString.copyFromUtf8(sb.toString())) + .setRowCount(rowCount) + .build() + response.setJsonBatch(batch) + responseObserver.onNext(response.build()) + sb.clear() + sb.append(row) + rowCount = 1 + } else { + // Case 2 - Append. + // Make sure to put the newline delimiters only between items and not at the end. + if (rowCount > 0) { + sb.append("\n") + } + sb.append(row) + rowCount += 1 + } + }) + + // If the last batch is not empty, send out the data to the client. + if (sb.size > 0) { + val response = proto.Response.newBuilder().setClientId(clientId) + val batch = proto.Response.JSONBatch + .newBuilder() + .setData(ByteString.copyFromUtf8(sb.toString())) + .setRowCount(rowCount) + .build() + response.setJsonBatch(batch) + responseObserver.onNext(response.build()) + } + responseObserver.onNext(sendMetricsToResponse(clientId, rows)) responseObserver.onCompleted() } diff --git a/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectDeduplicateSuite.scala b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectDeduplicateSuite.scala new file mode 100644 index 000000000000..88af60581ba2 --- /dev/null +++ b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectDeduplicateSuite.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.connect.planner + +import org.apache.spark.sql.{Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.expressions.AttributeReference +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} + +/** + * [[SparkConnectPlanTestWithSparkSession]] contains a SparkSession for the connect planner. + * + * It is not recommended to use Catalyst DSL along with this trait because `SharedSparkSession` + * has also defined implicits over Catalyst LogicalPlan which will cause ambiguity with the + * implicits defined in Catalyst DSL. + */ +trait SparkConnectPlanTestWithSparkSession extends SharedSparkSession with SparkConnectPlanTest { + override def getSession(): SparkSession = spark +} + +class SparkConnectDeduplicateSuite extends SparkConnectPlanTestWithSparkSession { + lazy val connectTestRelation = createLocalRelationProto( + Seq( + AttributeReference("id", IntegerType)(), + AttributeReference("key", StringType)(), + AttributeReference("value", StringType)())) + + lazy val sparkTestRelation = { + spark.createDataFrame( + new java.util.ArrayList[Row](), + StructType( + Seq( + StructField("id", IntegerType), + StructField("key", StringType), + StructField("value", StringType)))) + } + + test("Test basic deduplicate") { + val connectPlan = { + import org.apache.spark.sql.connect.dsl.plans._ + Dataset.ofRows(spark, transform(connectTestRelation.distinct())) + } + + val sparkPlan = sparkTestRelation.distinct() + comparePlans(connectPlan.queryExecution.analyzed, sparkPlan.queryExecution.analyzed, false) + + val connectPlan2 = { + import org.apache.spark.sql.connect.dsl.plans._ + Dataset.ofRows(spark, transform(connectTestRelation.deduplicate(Seq("key", "value")))) + } + val sparkPlan2 = sparkTestRelation.dropDuplicates(Seq("key", "value")) + comparePlans(connectPlan2.queryExecution.analyzed, sparkPlan2.queryExecution.analyzed, false) + } +} diff --git a/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala index fc3d219ec6ba..980e899c26ed 100644 --- a/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala +++ b/connector/connect/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala @@ -31,8 +31,11 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan * test cases. */ trait SparkConnectPlanTest { + + def getSession(): SparkSession = None.orNull + def transform(rel: proto.Relation): LogicalPlan = { - new SparkConnectPlanner(rel, None.orNull).transform() + new SparkConnectPlanner(rel, getSession()).transform() } def readRel: proto.Relation = @@ -72,8 +75,6 @@ trait SparkConnectPlanTest { */ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { - protected var spark: SparkSession = null - test("Simple Limit") { assertThrows[IndexOutOfBoundsException] { new SparkConnectPlanner( @@ -255,4 +256,37 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { assert(res.nodeName == "Aggregate") } + test("Invalid DataSource") { + val dataSource = proto.Read.DataSource.newBuilder() + + val e = intercept[InvalidPlanInput]( + transform( + proto.Relation + .newBuilder() + .setRead(proto.Read.newBuilder().setDataSource(dataSource)) + .build())) + assert(e.getMessage.contains("DataSource requires a format")) + } + + test("Test invalid deduplicate") { + val deduplicate = proto.Deduplicate + .newBuilder() + .setInput(readRel) + .setAllColumnsAsKeys(true) + .addColumnNames("test") + + val e = intercept[InvalidPlanInput] { + transform(proto.Relation.newBuilder.setDeduplicate(deduplicate).build()) + } + assert( + e.getMessage.contains("Cannot deduplicate on both all columns and a subset of columns")) + + val deduplicate2 = proto.Deduplicate + .newBuilder() + .setInput(readRel) + val e2 = intercept[InvalidPlanInput] { + transform(proto.Relation.newBuilder.setDeduplicate(deduplicate2).build()) + } + assert(e2.getMessage.contains("either deduplicate on all columns or a subset of columns")) + } } diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index 0515f128b8d6..b934c7f831a2 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -83,7 +83,6 @@ ${protobuf.version} compile - target/scala-${scala.binary.version}/classes @@ -110,6 +109,28 @@ + + com.github.os72 + protoc-jar-maven-plugin + 3.11.4 + + + + generate-test-sources + + run + + + com.google.protobuf:protoc:${protobuf.version} + ${protobuf.version} + + src/test/resources/protobuf + + test + + + + diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/CatalystDataToProtobuf.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/CatalystDataToProtobuf.scala index 145100268c23..b9f7907ea8ca 100644 --- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/CatalystDataToProtobuf.scala +++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/CatalystDataToProtobuf.scala @@ -25,17 +25,17 @@ import org.apache.spark.sql.types.{BinaryType, DataType} private[protobuf] case class CatalystDataToProtobuf( child: Expression, - descFilePath: String, - messageName: String) + messageName: String, + descFilePath: Option[String] = None) extends UnaryExpression { override def dataType: DataType = BinaryType - @transient private lazy val protoType = - ProtobufUtils.buildDescriptor(descFilePath, messageName) + @transient private lazy val protoDescriptor = + ProtobufUtils.buildDescriptor(messageName, descFilePathOpt = descFilePath) @transient private lazy val serializer = - new ProtobufSerializer(child.dataType, protoType, child.nullable) + new ProtobufSerializer(child.dataType, protoDescriptor, child.nullable) override def nullSafeEval(input: Any): Any = { val dynamicMessage = serializer.serialize(input).asInstanceOf[DynamicMessage] diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufDataToCatalyst.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufDataToCatalyst.scala index f08f87679972..cad2442f10c1 100644 --- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufDataToCatalyst.scala +++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/ProtobufDataToCatalyst.scala @@ -31,9 +31,9 @@ import org.apache.spark.sql.types.{AbstractDataType, BinaryType, DataType, Struc private[protobuf] case class ProtobufDataToCatalyst( child: Expression, - descFilePath: String, messageName: String, - options: Map[String, String]) + descFilePath: Option[String] = None, + options: Map[String, String] = Map.empty) extends UnaryExpression with ExpectsInputTypes { @@ -55,10 +55,14 @@ private[protobuf] case class ProtobufDataToCatalyst( private lazy val protobufOptions = ProtobufOptions(options) @transient private lazy val messageDescriptor = - ProtobufUtils.buildDescriptor(descFilePath, messageName) + ProtobufUtils.buildDescriptor(messageName, descFilePath) + // TODO: Avoid carrying the file name. Read the contents of descriptor file only once + // at the start. Rest of the runs should reuse the buffer. Otherwise, it could + // cause inconsistencies if the file contents are changed the user after a few days. + // Same for the write side in [[CatalystDataToProtobuf]]. @transient private lazy val fieldsNumbers = - messageDescriptor.getFields.asScala.map(f => f.getNumber) + messageDescriptor.getFields.asScala.map(f => f.getNumber).toSet @transient private lazy val deserializer = new ProtobufDeserializer(messageDescriptor, dataType) @@ -108,18 +112,18 @@ private[protobuf] case class ProtobufDataToCatalyst( val binary = input.asInstanceOf[Array[Byte]] try { result = DynamicMessage.parseFrom(messageDescriptor, binary) - val unknownFields = result.getUnknownFields - if (!unknownFields.asMap().isEmpty) { - unknownFields.asMap().keySet().asScala.map { number => - { - if (fieldsNumbers.contains(number)) { - return handleException( - new Throwable(s"Type mismatch encountered for field:" + - s" ${messageDescriptor.getFields.get(number)}")) - } - } - } + // If the Java class is available, it is likely more efficient to parse with it than using + // DynamicMessage. Can consider it in the future if parsing overhead is noticeable. + + result.getUnknownFields.asMap().keySet().asScala.find(fieldsNumbers.contains(_)) match { + case Some(number) => + // Unknown fields contain a field with same number as a known field. Must be due to + // mismatch of schema between writer and reader here. + throw new IllegalArgumentException(s"Type mismatch encountered for field:" + + s" ${messageDescriptor.getFields.get(number)}") + case None => } + val deserialized = deserializer.deserialize(result) assert( deserialized.isDefined, diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala index 283d1ca8c412..af30de40dad0 100644 --- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala +++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/functions.scala @@ -33,20 +33,21 @@ object functions { * * @param data * the binary column. - * @param descFilePath - * the protobuf descriptor in Message GeneratedMessageV3 format. * @param messageName * the protobuf message name to look for in descriptorFile. + * @param descFilePath + * the protobuf descriptor in Message GeneratedMessageV3 format. * @since 3.4.0 */ @Experimental def from_protobuf( data: Column, - descFilePath: String, messageName: String, + descFilePath: String, options: java.util.Map[String, String]): Column = { new Column( - ProtobufDataToCatalyst(data.expr, descFilePath, messageName, options.asScala.toMap)) + ProtobufDataToCatalyst(data.expr, messageName, Some(descFilePath), options.asScala.toMap) + ) } /** @@ -57,15 +58,34 @@ object functions { * * @param data * the binary column. - * @param descFilePath - * the protobuf descriptor in Message GeneratedMessageV3 format. * @param messageName * the protobuf MessageName to look for in descriptorFile. + * @param descFilePath + * the protobuf descriptor in Message GeneratedMessageV3 format. * @since 3.4.0 */ @Experimental - def from_protobuf(data: Column, descFilePath: String, messageName: String): Column = { - new Column(ProtobufDataToCatalyst(data.expr, descFilePath, messageName, Map.empty)) + def from_protobuf(data: Column, messageName: String, descFilePath: String): Column = { + new Column(ProtobufDataToCatalyst(data.expr, messageName, descFilePath = Some(descFilePath))) + // TODO: Add an option for user to provide descriptor file content as a buffer. This + // gives flexibility in how the content is fetched. + } + + /** + * Converts a binary column of Protobuf format into its corresponding catalyst value. The + * specified schema must match actual schema of the read data, otherwise the behavior is + * undefined: it may fail or return arbitrary result. To deserialize the data with a compatible + * and evolved schema, the expected Protobuf schema can be set via the option protoSchema. + * + * @param data + * the binary column. + * @param messageClassName + * The Protobuf class name. E.g. org.spark.examples.protobuf.ExampleEvent. + * @since 3.4.0 + */ + @Experimental + def from_protobuf(data: Column, messageClassName: String): Column = { + new Column(ProtobufDataToCatalyst(data.expr, messageClassName)) } /** @@ -73,14 +93,28 @@ object functions { * * @param data * the data column. - * @param descFilePath - * the protobuf descriptor in Message GeneratedMessageV3 format. * @param messageName * the protobuf MessageName to look for in descriptorFile. + * @param descFilePath + * the protobuf descriptor in Message GeneratedMessageV3 format. + * @since 3.4.0 + */ + @Experimental + def to_protobuf(data: Column, messageName: String, descFilePath: String): Column = { + new Column(CatalystDataToProtobuf(data.expr, messageName, Some(descFilePath))) + } + + /** + * Converts a column into binary of protobuf format. + * + * @param data + * the data column. + * @param messageClassName + * The Protobuf class name. E.g. org.spark.examples.protobuf.ExampleEvent. * @since 3.4.0 */ @Experimental - def to_protobuf(data: Column, descFilePath: String, messageName: String): Column = { - new Column(CatalystDataToProtobuf(data.expr, descFilePath, messageName)) + def to_protobuf(data: Column, messageClassName: String): Column = { + new Column(CatalystDataToProtobuf(data.expr, messageClassName)) } } diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala index 5ad043142a2d..fa2ec9b7cd46 100644 --- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala +++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala @@ -22,13 +22,14 @@ import java.util.Locale import scala.collection.JavaConverters._ -import com.google.protobuf.{DescriptorProtos, Descriptors, InvalidProtocolBufferException} +import com.google.protobuf.{DescriptorProtos, Descriptors, InvalidProtocolBufferException, Message} import com.google.protobuf.Descriptors.{Descriptor, FieldDescriptor} import org.apache.spark.internal.Logging import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.protobuf.utils.SchemaConverters.IncompatibleSchemaException import org.apache.spark.sql.types._ +import org.apache.spark.util.Utils private[sql] object ProtobufUtils extends Logging { @@ -132,23 +133,63 @@ private[sql] object ProtobufUtils extends Logging { } } - def buildDescriptor(descFilePath: String, messageName: String): Descriptor = { - val fileDescriptor: Descriptors.FileDescriptor = parseFileDescriptor(descFilePath) - var result: Descriptors.Descriptor = null; + /** + * Builds Protobuf message descriptor either from the Java class or from serialized descriptor + * read from the file. + * @param messageName + * Protobuf message name or Java class name. + * @param descFilePathOpt + * When the file name set, the descriptor and it's dependencies are read from the file. Other + * the `messageName` is treated as Java class name. + * @return + */ + def buildDescriptor(messageName: String, descFilePathOpt: Option[String]): Descriptor = { + descFilePathOpt match { + case Some(filePath) => buildDescriptor(descFilePath = filePath, messageName) + case None => buildDescriptorFromJavaClass(messageName) + } + } - for (descriptor <- fileDescriptor.getMessageTypes.asScala) { - if (descriptor.getName().equals(messageName)) { - result = descriptor - } + /** + * Loads the given protobuf class and returns Protobuf descriptor for it. + */ + def buildDescriptorFromJavaClass(protobufClassName: String): Descriptor = { + val protobufClass = try { + Utils.classForName(protobufClassName) + } catch { + case _: ClassNotFoundException => + val hasDots = protobufClassName.contains(".") + throw new IllegalArgumentException( + s"Could not load Protobuf class with name '$protobufClassName'" + + (if (hasDots) "" else ". Ensure the class name includes package prefix.") + ) + } + + if (!classOf[Message].isAssignableFrom(protobufClass)) { + throw new IllegalArgumentException(s"$protobufClassName is not a Protobuf message type") + // TODO: Need to support V2. This might work with V2 classes too. + } + + // Extract the descriptor from Protobuf message. + protobufClass + .getDeclaredMethod("getDescriptor") + .invoke(null) + .asInstanceOf[Descriptor] + } + + def buildDescriptor(descFilePath: String, messageName: String): Descriptor = { + val descriptor = parseFileDescriptor(descFilePath).getMessageTypes.asScala.find { desc => + desc.getName == messageName || desc.getFullName == messageName } - if (null == result) { - throw new RuntimeException("Unable to locate Message '" + messageName + "' in Descriptor"); + descriptor match { + case Some(d) => d + case None => + throw new RuntimeException(s"Unable to locate Message '$messageName' in Descriptor") } - result } - def parseFileDescriptor(descFilePath: String): Descriptors.FileDescriptor = { + private def parseFileDescriptor(descFilePath: String): Descriptors.FileDescriptor = { var fileDescriptorSet: DescriptorProtos.FileDescriptorSet = null try { val dscFile = new BufferedInputStream(new FileInputStream(descFilePath)) diff --git a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/SchemaConverters.scala b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/SchemaConverters.scala index e385b816abe7..4fca06fb5d8b 100644 --- a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/SchemaConverters.scala +++ b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/SchemaConverters.scala @@ -66,6 +66,10 @@ object SchemaConverters { Some(DayTimeIntervalType.defaultConcreteType) case MESSAGE if fd.getMessageType.getName == "Timestamp" => Some(TimestampType) + // FIXME: Is the above accurate? Users can have protos named "Timestamp" but are not + // expected to be TimestampType in Spark. How about verifying fields? + // Same for "Duration". Only the Timestamp & Duration protos defined in + // google.protobuf package should default to corresponding Catalylist types. case MESSAGE if fd.isRepeated && fd.getMessageType.getOptions.hasMapEntry => var keyType: DataType = NullType var valueType: DataType = NullType diff --git a/connector/protobuf/src/test/resources/protobuf/catalyst_types.proto b/connector/protobuf/src/test/resources/protobuf/catalyst_types.proto index 54e6bc18df15..1deb193438c2 100644 --- a/connector/protobuf/src/test/resources/protobuf/catalyst_types.proto +++ b/connector/protobuf/src/test/resources/protobuf/catalyst_types.proto @@ -19,9 +19,11 @@ syntax = "proto3"; -package org.apache.spark.sql.protobuf; +package org.apache.spark.sql.protobuf.protos; option java_outer_classname = "CatalystTypes"; +// TODO: import one or more protobuf files. + message BooleanMsg { bool bool_type = 1; } diff --git a/connector/protobuf/src/test/resources/protobuf/functions_suite.proto b/connector/protobuf/src/test/resources/protobuf/functions_suite.proto index f38c041b799e..60f8c2621415 100644 --- a/connector/protobuf/src/test/resources/protobuf/functions_suite.proto +++ b/connector/protobuf/src/test/resources/protobuf/functions_suite.proto @@ -20,7 +20,7 @@ syntax = "proto3"; -package org.apache.spark.sql.protobuf; +package org.apache.spark.sql.protobuf.protos; option java_outer_classname = "SimpleMessageProtos"; @@ -119,7 +119,7 @@ message SimpleMessageEnum { string key = 1; string value = 2; enum NestedEnum { - ESTED_NOTHING = 0; + ESTED_NOTHING = 0; // TODO: Fix the name. NESTED_FIRST = 1; NESTED_SECOND = 2; } diff --git a/connector/protobuf/src/test/resources/protobuf/serde_suite.proto b/connector/protobuf/src/test/resources/protobuf/serde_suite.proto index 1e3065259aa0..a7459213a87b 100644 --- a/connector/protobuf/src/test/resources/protobuf/serde_suite.proto +++ b/connector/protobuf/src/test/resources/protobuf/serde_suite.proto @@ -20,11 +20,11 @@ syntax = "proto3"; -package org.apache.spark.sql.protobuf; -option java_outer_classname = "SimpleMessageProtos"; +package org.apache.spark.sql.protobuf.protos; +option java_outer_classname = "SerdeSuiteProtos"; /* Clean Message*/ -message BasicMessage { +message SerdeBasicMessage { Foo foo = 1; } diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala index b730ebb4fea8..19774a2ad07e 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.{RandomDataGenerator, Row} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, NoopFilters, OrderedFilters, StructFilters} import org.apache.spark.sql.catalyst.expressions.{ExpressionEvalHelper, GenericInternalRow, Literal} import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData, MapData} +import org.apache.spark.sql.protobuf.protos.CatalystTypes.BytesMsg import org.apache.spark.sql.protobuf.utils.{ProtobufUtils, SchemaConverters} import org.apache.spark.sql.sources.{EqualTo, Not} import org.apache.spark.sql.test.SharedSparkSession @@ -35,18 +36,32 @@ class ProtobufCatalystDataConversionSuite with SharedSparkSession with ExpressionEvalHelper { - private def checkResult( + private val testFileDesc = testFile("protobuf/catalyst_types.desc").replace("file:/", "/") + private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.CatalystTypes$" + + private def checkResultWithEval( data: Literal, descFilePath: String, messageName: String, expected: Any): Unit = { - checkEvaluation( - ProtobufDataToCatalyst( - CatalystDataToProtobuf(data, descFilePath, messageName), - descFilePath, - messageName, - Map.empty), - prepareExpectedResult(expected)) + + withClue("(Eval check with Java class name)") { + val className = s"$javaClassNamePrefix$messageName" + checkEvaluation( + ProtobufDataToCatalyst( + CatalystDataToProtobuf(data, className), + className, + descFilePath = None), + prepareExpectedResult(expected)) + } + withClue("(Eval check with descriptor file)") { + checkEvaluation( + ProtobufDataToCatalyst( + CatalystDataToProtobuf(data, messageName, Some(descFilePath)), + messageName, + descFilePath = Some(descFilePath)), + prepareExpectedResult(expected)) + } } protected def checkUnsupportedRead( @@ -55,10 +70,11 @@ class ProtobufCatalystDataConversionSuite actualSchema: String, badSchema: String): Unit = { - val binary = CatalystDataToProtobuf(data, descFilePath, actualSchema) + val binary = CatalystDataToProtobuf(data, actualSchema, Some(descFilePath)) intercept[Exception] { - ProtobufDataToCatalyst(binary, descFilePath, badSchema, Map("mode" -> "FAILFAST")).eval() + ProtobufDataToCatalyst(binary, badSchema, Some(descFilePath), Map("mode" -> "FAILFAST")) + .eval() } val expected = { @@ -73,7 +89,7 @@ class ProtobufCatalystDataConversionSuite } checkEvaluation( - ProtobufDataToCatalyst(binary, descFilePath, badSchema, Map("mode" -> "PERMISSIVE")), + ProtobufDataToCatalyst(binary, badSchema, Some(descFilePath), Map("mode" -> "PERMISSIVE")), expected) } @@ -99,26 +115,32 @@ class ProtobufCatalystDataConversionSuite StructType(StructField("bytes_type", BinaryType, nullable = true) :: Nil), StructType(StructField("string_type", StringType, nullable = true) :: Nil)) - private val catalystTypesToProtoMessages: Map[DataType, String] = Map( - IntegerType -> "IntegerMsg", - DoubleType -> "DoubleMsg", - FloatType -> "FloatMsg", - BinaryType -> "BytesMsg", - StringType -> "StringMsg") + private val catalystTypesToProtoMessages: Map[DataType, (String, Any)] = Map( + IntegerType -> ("IntegerMsg", 0), + DoubleType -> ("DoubleMsg", 0.0d), + FloatType -> ("FloatMsg", 0.0f), + BinaryType -> ("BytesMsg", ByteString.empty().toByteArray), + StringType -> ("StringMsg", "")) testingTypes.foreach { dt => val seed = 1 + scala.util.Random.nextInt((1024 - 1) + 1) - val filePath = testFile("protobuf/catalyst_types.desc").replace("file:/", "/") test(s"single $dt with seed $seed") { + + val (messageName, defaultValue) = catalystTypesToProtoMessages(dt.fields(0).dataType) + val rand = new scala.util.Random(seed) - val data = RandomDataGenerator.forType(dt, rand = rand).get.apply() + val generator = RandomDataGenerator.forType(dt, rand = rand).get + var data = generator() + while (data.asInstanceOf[Row].get(0) == defaultValue) // Do not use default values, since + data = generator() // from_protobuf() returns null in v3. + val converter = CatalystTypeConverters.createToCatalystConverter(dt) val input = Literal.create(converter(data), dt) - checkResult( + checkResultWithEval( input, - filePath, - catalystTypesToProtoMessages(dt.fields(0).dataType), + testFileDesc, + messageName, input.eval()) } } @@ -137,6 +159,15 @@ class ProtobufCatalystDataConversionSuite val dynMsg = DynamicMessage.parseFrom(descriptor, data.toByteArray) val deserialized = deserializer.deserialize(dynMsg) + + // Verify Java class deserializer matches with descriptor based serializer. + val javaDescriptor = ProtobufUtils + .buildDescriptorFromJavaClass(s"$javaClassNamePrefix$messageName") + assert(dataType == SchemaConverters.toSqlType(javaDescriptor).dataType) + val javaDeserialized = new ProtobufDeserializer(javaDescriptor, dataType, filters) + .deserialize(DynamicMessage.parseFrom(javaDescriptor, data.toByteArray)) + assert(deserialized == javaDeserialized) + expected match { case None => assert(deserialized.isEmpty) case Some(d) => @@ -145,7 +176,6 @@ class ProtobufCatalystDataConversionSuite } test("Handle unsupported input of message type") { - val testFileDesc = testFile("protobuf/catalyst_types.desc").replace("file:/", "/") val actualSchema = StructType( Seq( StructField("col_0", StringType, nullable = false), @@ -165,7 +195,6 @@ class ProtobufCatalystDataConversionSuite test("filter push-down to Protobuf deserializer") { - val testFileDesc = testFile("protobuf/catalyst_types.desc").replace("file:/", "/") val sqlSchema = new StructType() .add("name", "string") .add("age", "int") @@ -196,17 +225,23 @@ class ProtobufCatalystDataConversionSuite test("ProtobufDeserializer with binary type") { - val testFileDesc = testFile("protobuf/catalyst_types.desc").replace("file:/", "/") val bb = java.nio.ByteBuffer.wrap(Array[Byte](97, 48, 53)) - val descriptor = ProtobufUtils.buildDescriptor(testFileDesc, "BytesMsg") - - val dynamicMessage = DynamicMessage - .newBuilder(descriptor) - .setField(descriptor.findFieldByName("bytes_type"), ByteString.copyFrom(bb)) + val bytesProto = BytesMsg + .newBuilder() + .setBytesType(ByteString.copyFrom(bb)) .build() val expected = InternalRow(Array[Byte](97, 48, 53)) - checkDeserialization(testFileDesc, "BytesMsg", dynamicMessage, Some(expected)) + checkDeserialization(testFileDesc, "BytesMsg", bytesProto, Some(expected)) + } + + test("Full names for message using descriptor file") { + val withShortName = ProtobufUtils.buildDescriptor(testFileDesc, "BytesMsg") + assert(withShortName.findFieldByName("bytes_type") != null) + + val withFullName = ProtobufUtils.buildDescriptor( + testFileDesc, "org.apache.spark.sql.protobuf.BytesMsg") + assert(withFullName.findFieldByName("bytes_type") != null) } } diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala index 4e9bc1c1c287..72280fb0d9e2 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala @@ -23,8 +23,10 @@ import scala.collection.JavaConverters._ import com.google.protobuf.{ByteString, DynamicMessage} -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{Column, QueryTest, Row} import org.apache.spark.sql.functions.{lit, struct} +import org.apache.spark.sql.protobuf.protos.SimpleMessageProtos.SimpleMessageRepeated +import org.apache.spark.sql.protobuf.protos.SimpleMessageProtos.SimpleMessageRepeated.NestedEnum import org.apache.spark.sql.protobuf.utils.ProtobufUtils import org.apache.spark.sql.protobuf.utils.SchemaConverters.IncompatibleSchemaException import org.apache.spark.sql.test.SharedSparkSession @@ -35,6 +37,39 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri import testImplicits._ val testFileDesc = testFile("protobuf/functions_suite.desc").replace("file:/", "/") + private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SimpleMessageProtos$" + + /** + * Runs the given closure twice. Once with descriptor file and second time with Java class name. + */ + private def checkWithFileAndClassName(messageName: String)( + fn: (String, Option[String]) => Unit): Unit = { + withClue("(With descriptor file)") { + fn(messageName, Some(testFileDesc)) + } + withClue("(With Java class name)") { + fn(s"$javaClassNamePrefix$messageName", None) + } + } + + // A wrapper to invoke the right variable of from_protobuf() depending on arguments. + private def from_protobuf_wrapper( + col: Column, messageName: String, descFilePathOpt: Option[String]): Column = { + descFilePathOpt match { + case Some(descFilePath) => functions.from_protobuf(col, messageName, descFilePath) + case None => functions.from_protobuf(col, messageName) + } + } + + // A wrapper to invoke the right variable of to_protobuf() depending on arguments. + private def to_protobuf_wrapper( + col: Column, messageName: String, descFilePathOpt: Option[String]): Column = { + descFilePathOpt match { + case Some(descFilePath) => functions.to_protobuf(col, messageName, descFilePath) + case None => functions.to_protobuf(col, messageName) + } + } + test("roundtrip in to_protobuf and from_protobuf - struct") { val df = spark @@ -56,44 +91,45 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri lit(1202.00).cast(org.apache.spark.sql.types.FloatType).as("float_value"), lit(true).as("bool_value"), lit("0".getBytes).as("bytes_value")).as("SimpleMessage")) - val protoStructDF = df.select( - functions.to_protobuf($"SimpleMessage", testFileDesc, "SimpleMessage").as("proto")) - val actualDf = protoStructDF.select( - functions.from_protobuf($"proto", testFileDesc, "SimpleMessage").as("proto.*")) - checkAnswer(actualDf, df) + + checkWithFileAndClassName("SimpleMessage") { + case (name, descFilePathOpt) => + val protoStructDF = df.select( + to_protobuf_wrapper($"SimpleMessage", name, descFilePathOpt).as("proto")) + val actualDf = protoStructDF.select( + from_protobuf_wrapper($"proto", name, descFilePathOpt).as("proto.*")) + checkAnswer(actualDf, df) + } } test("roundtrip in from_protobuf and to_protobuf - Repeated") { - val descriptor = ProtobufUtils.buildDescriptor(testFileDesc, "SimpleMessageRepeated") - val dynamicMessage = DynamicMessage - .newBuilder(descriptor) - .setField(descriptor.findFieldByName("key"), "key") - .setField(descriptor.findFieldByName("value"), "value") - .addRepeatedField(descriptor.findFieldByName("rbool_value"), false) - .addRepeatedField(descriptor.findFieldByName("rbool_value"), true) - .addRepeatedField(descriptor.findFieldByName("rdouble_value"), 1092092.654d) - .addRepeatedField(descriptor.findFieldByName("rdouble_value"), 1092093.654d) - .addRepeatedField(descriptor.findFieldByName("rfloat_value"), 10903.0f) - .addRepeatedField(descriptor.findFieldByName("rfloat_value"), 10902.0f) - .addRepeatedField( - descriptor.findFieldByName("rnested_enum"), - descriptor.findEnumTypeByName("NestedEnum").findValueByName("ESTED_NOTHING")) - .addRepeatedField( - descriptor.findFieldByName("rnested_enum"), - descriptor.findEnumTypeByName("NestedEnum").findValueByName("NESTED_FIRST")) + val protoMessage = SimpleMessageRepeated + .newBuilder() + .setKey("key") + .setValue("value") + .addRboolValue(false) + .addRboolValue(true) + .addRdoubleValue(1092092.654d) + .addRdoubleValue(1092093.654d) + .addRfloatValue(10903.0f) + .addRfloatValue(10902.0f) + .addRnestedEnum(NestedEnum.ESTED_NOTHING) + .addRnestedEnum(NestedEnum.NESTED_FIRST) .build() - val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "SimpleMessageRepeated").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "SimpleMessageRepeated").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions - .from_protobuf($"value_to", testFileDesc, "SimpleMessageRepeated") - .as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + val df = Seq(protoMessage.toByteArray).toDF("value") + + checkWithFileAndClassName("SimpleMessageRepeated") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("roundtrip in from_protobuf and to_protobuf - Repeated Message Once") { @@ -120,13 +156,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "RepeatedMessage").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "RepeatedMessage").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions.from_protobuf($"value_to", testFileDesc, "RepeatedMessage").as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + + checkWithFileAndClassName("RepeatedMessage") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("roundtrip in from_protobuf and to_protobuf - Repeated Message Twice") { @@ -167,13 +207,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "RepeatedMessage").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "RepeatedMessage").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions.from_protobuf($"value_to", testFileDesc, "RepeatedMessage").as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + + checkWithFileAndClassName("RepeatedMessage") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("roundtrip in from_protobuf and to_protobuf - Map") { @@ -257,13 +301,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "SimpleMessageMap").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "SimpleMessageMap").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions.from_protobuf($"value_to", testFileDesc, "SimpleMessageMap").as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + + checkWithFileAndClassName("SimpleMessageMap") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("roundtrip in from_protobuf and to_protobuf - Enum") { @@ -289,13 +337,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "SimpleMessageEnum").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "SimpleMessageEnum").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions.from_protobuf($"value_to", testFileDesc, "SimpleMessageEnum").as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + + checkWithFileAndClassName("SimpleMessageEnum") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("roundtrip in from_protobuf and to_protobuf - Multiple Message") { @@ -320,13 +372,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(dynamicMessage.toByteArray).toDF("value") - val fromProtoDF = df.select( - functions.from_protobuf($"value", testFileDesc, "MultipleExample").as("value_from")) - val toProtoDF = fromProtoDF.select( - functions.to_protobuf($"value_from", testFileDesc, "MultipleExample").as("value_to")) - val toFromProtoDF = toProtoDF.select( - functions.from_protobuf($"value_to", testFileDesc, "MultipleExample").as("value_to_from")) - checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + + checkWithFileAndClassName("MultipleExample") { + case (name, descFilePathOpt) => + val fromProtoDF = df.select( + from_protobuf_wrapper($"value", name, descFilePathOpt).as("value_from")) + val toProtoDF = fromProtoDF.select( + to_protobuf_wrapper($"value_from", name, descFilePathOpt).as("value_to")) + val toFromProtoDF = toProtoDF.select( + from_protobuf_wrapper($"value_to", name, descFilePathOpt).as("value_to_from")) + checkAnswer(fromProtoDF.select($"value_from.*"), toFromProtoDF.select($"value_to_from.*")) + } } test("Handle recursive fields in Protobuf schema, A->B->A") { @@ -352,15 +408,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri val df = Seq(messageB.toByteArray).toDF("messageB") - val e = intercept[IncompatibleSchemaException] { - df.select( - functions.from_protobuf($"messageB", testFileDesc, "recursiveB").as("messageFromProto")) - .show() + checkWithFileAndClassName("recursiveB") { + case (name, descFilePathOpt) => + val e = intercept[IncompatibleSchemaException] { + df.select( + from_protobuf_wrapper($"messageB", name, descFilePathOpt).as("messageFromProto")) + .show() + } + assert(e.getMessage.contains( + "Found recursive reference in Protobuf schema, which can not be processed by Spark:" + )) } - val expectedMessage = s""" - |Found recursive reference in Protobuf schema, which can not be processed by Spark: - |org.apache.spark.sql.protobuf.recursiveB.messageA""".stripMargin - assert(e.getMessage == expectedMessage) } test("Handle recursive fields in Protobuf schema, C->D->Array(C)") { @@ -386,16 +444,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri val df = Seq(messageD.toByteArray).toDF("messageD") - val e = intercept[IncompatibleSchemaException] { - df.select( - functions.from_protobuf($"messageD", testFileDesc, "recursiveD").as("messageFromProto")) - .show() + checkWithFileAndClassName("recursiveD") { + case (name, descFilePathOpt) => + val e = intercept[IncompatibleSchemaException] { + df.select( + from_protobuf_wrapper($"messageD", name, descFilePathOpt).as("messageFromProto")) + .show() + } + assert(e.getMessage.contains( + "Found recursive reference in Protobuf schema, which can not be processed by Spark:" + )) } - val expectedMessage = - s""" - |Found recursive reference in Protobuf schema, which can not be processed by Spark: - |org.apache.spark.sql.protobuf.recursiveD.messageC""".stripMargin - assert(e.getMessage == expectedMessage) } test("Handle extra fields : oldProducer -> newConsumer") { @@ -411,17 +470,17 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri val df = Seq(oldProducerMessage.toByteArray).toDF("oldProducerData") val fromProtoDf = df.select( functions - .from_protobuf($"oldProducerData", testFileDesc, "newConsumer") + .from_protobuf($"oldProducerData", "newConsumer", testFileDesc) .as("fromProto")) val toProtoDf = fromProtoDf.select( functions - .to_protobuf($"fromProto", testFileDesc, "newConsumer") + .to_protobuf($"fromProto", "newConsumer", testFileDesc) .as("toProto")) val toProtoDfToFromProtoDf = toProtoDf.select( functions - .from_protobuf($"toProto", testFileDesc, "newConsumer") + .from_protobuf($"toProto", "newConsumer", testFileDesc) .as("toProtoToFromProto")) val actualFieldNames = @@ -452,7 +511,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri val df = Seq(newProducerMessage.toByteArray).toDF("newProducerData") val fromProtoDf = df.select( functions - .from_protobuf($"newProducerData", testFileDesc, "oldConsumer") + .from_protobuf($"newProducerData", "oldConsumer", testFileDesc) .as("oldConsumerProto")) val expectedFieldNames = oldConsumer.getFields.asScala.map(f => f.getName) @@ -481,8 +540,9 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri )), schema ) + val toProtobuf = inputDf.select( - functions.to_protobuf($"requiredMsg", testFileDesc, "requiredMsg") + functions.to_protobuf($"requiredMsg", "requiredMsg", testFileDesc) .as("to_proto")) val binary = toProtobuf.take(1).toSeq(0).get(0).asInstanceOf[Array[Byte]] @@ -498,7 +558,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri assert(actualMessage.getField(messageDescriptor.findFieldByName("col_3")) == 0) val fromProtoDf = toProtobuf.select( - functions.from_protobuf($"to_proto", testFileDesc, "requiredMsg") as 'from_proto) + functions.from_protobuf($"to_proto", "requiredMsg", testFileDesc) as 'from_proto) assert(fromProtoDf.select("from_proto.key").take(1).toSeq(0).get(0) == inputDf.select("requiredMsg.key").take(1).toSeq(0).get(0)) @@ -526,16 +586,20 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri .build() val df = Seq(basicMessage.toByteArray).toDF("value") - val resultFrom = df - .select(functions.from_protobuf($"value", testFileDesc, "BasicMessage") as 'sample) - .where("sample.string_value == \"slam\"") - val resultToFrom = resultFrom - .select(functions.to_protobuf($"sample", testFileDesc, "BasicMessage") as 'value) - .select(functions.from_protobuf($"value", testFileDesc, "BasicMessage") as 'sample) - .where("sample.string_value == \"slam\"") + checkWithFileAndClassName("BasicMessage") { + case (name, descFilePathOpt) => + val resultFrom = df + .select(from_protobuf_wrapper($"value", name, descFilePathOpt) as 'sample) + .where("sample.string_value == \"slam\"") + + val resultToFrom = resultFrom + .select(to_protobuf_wrapper($"sample", name, descFilePathOpt) as 'value) + .select(from_protobuf_wrapper($"value", name, descFilePathOpt) as 'sample) + .where("sample.string_value == \"slam\"") - assert(resultFrom.except(resultToFrom).isEmpty) + assert(resultFrom.except(resultToFrom).isEmpty) + } } test("Handle TimestampType between to_protobuf and from_protobuf") { @@ -556,22 +620,24 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri schema ) - val toProtoDf = inputDf - .select(functions.to_protobuf($"timeStampMsg", testFileDesc, "timeStampMsg") as 'to_proto) + checkWithFileAndClassName("timeStampMsg") { + case (name, descFilePathOpt) => + val toProtoDf = inputDf + .select(to_protobuf_wrapper($"timeStampMsg", name, descFilePathOpt) as 'to_proto) - val fromProtoDf = toProtoDf - .select(functions.from_protobuf($"to_proto", testFileDesc, "timeStampMsg") as 'timeStampMsg) - fromProtoDf.show(truncate = false) + val fromProtoDf = toProtoDf + .select(from_protobuf_wrapper($"to_proto", name, descFilePathOpt) as 'timeStampMsg) - val actualFields = fromProtoDf.schema.fields.toList - val expectedFields = inputDf.schema.fields.toList + val actualFields = fromProtoDf.schema.fields.toList + val expectedFields = inputDf.schema.fields.toList - assert(actualFields.size === expectedFields.size) - assert(actualFields === expectedFields) - assert(fromProtoDf.select("timeStampMsg.key").take(1).toSeq(0).get(0) - === inputDf.select("timeStampMsg.key").take(1).toSeq(0).get(0)) - assert(fromProtoDf.select("timeStampMsg.stmp").take(1).toSeq(0).get(0) - === inputDf.select("timeStampMsg.stmp").take(1).toSeq(0).get(0)) + assert(actualFields.size === expectedFields.size) + assert(actualFields === expectedFields) + assert(fromProtoDf.select("timeStampMsg.key").take(1).toSeq(0).get(0) + === inputDf.select("timeStampMsg.key").take(1).toSeq(0).get(0)) + assert(fromProtoDf.select("timeStampMsg.stmp").take(1).toSeq(0).get(0) + === inputDf.select("timeStampMsg.stmp").take(1).toSeq(0).get(0)) + } } test("Handle DayTimeIntervalType between to_protobuf and from_protobuf") { @@ -595,21 +661,23 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri schema ) - val toProtoDf = inputDf - .select(functions.to_protobuf($"durationMsg", testFileDesc, "durationMsg") as 'to_proto) + checkWithFileAndClassName("durationMsg") { + case (name, descFilePathOpt) => + val toProtoDf = inputDf + .select(to_protobuf_wrapper($"durationMsg", name, descFilePathOpt) as 'to_proto) - val fromProtoDf = toProtoDf - .select(functions.from_protobuf($"to_proto", testFileDesc, "durationMsg") as 'durationMsg) + val fromProtoDf = toProtoDf + .select(from_protobuf_wrapper($"to_proto", name, descFilePathOpt) as 'durationMsg) - val actualFields = fromProtoDf.schema.fields.toList - val expectedFields = inputDf.schema.fields.toList - - assert(actualFields.size === expectedFields.size) - assert(actualFields === expectedFields) - assert(fromProtoDf.select("durationMsg.key").take(1).toSeq(0).get(0) - === inputDf.select("durationMsg.key").take(1).toSeq(0).get(0)) - assert(fromProtoDf.select("durationMsg.duration").take(1).toSeq(0).get(0) - === inputDf.select("durationMsg.duration").take(1).toSeq(0).get(0)) + val actualFields = fromProtoDf.schema.fields.toList + val expectedFields = inputDf.schema.fields.toList + assert(actualFields.size === expectedFields.size) + assert(actualFields === expectedFields) + assert(fromProtoDf.select("durationMsg.key").take(1).toSeq(0).get(0) + === inputDf.select("durationMsg.key").take(1).toSeq(0).get(0)) + assert(fromProtoDf.select("durationMsg.duration").take(1).toSeq(0).get(0) + === inputDf.select("durationMsg.duration").take(1).toSeq(0).get(0)) + } } } diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala index 37c59743e771..efc02524e68d 100644 --- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala +++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala @@ -36,6 +36,7 @@ class ProtobufSerdeSuite extends SharedSparkSession { import ProtoSerdeSuite.MatchType._ val testFileDesc = testFile("protobuf/serde_suite.desc").replace("file:/", "/") + private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SerdeSuiteProtos$" test("Test basic conversion") { withFieldMatchType { fieldMatch => @@ -96,7 +97,9 @@ class ProtobufSerdeSuite extends SharedSparkSession { } test("Fail to convert with deeply nested field type mismatch") { - val protoFile = ProtobufUtils.buildDescriptor(testFileDesc, "MissMatchTypeInDeepNested") + val protoFile = ProtobufUtils.buildDescriptorFromJavaClass( + s"${javaClassNamePrefix}MissMatchTypeInDeepNested" + ) val catalyst = new StructType().add("top", CATALYST_STRUCT) withFieldMatchType { fieldMatch => @@ -105,8 +108,8 @@ class ProtobufSerdeSuite extends SharedSparkSession { Deserializer, fieldMatch, s"Cannot convert Protobuf field 'top.foo.bar' to SQL field 'top.foo.bar' because schema " + - s"is incompatible (protoType = org.apache.spark.sql.protobuf.TypeMiss.bar " + - s"LABEL_OPTIONAL LONG INT64, sqlType = INT)".stripMargin, + s"is incompatible (protoType = org.apache.spark.sql.protobuf.protos.TypeMiss.bar " + + s"LABEL_OPTIONAL LONG INT64, sqlType = INT)", catalyst) assertFailedConversionMessage( diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index 0cfb6861c776..1e9519dd89a1 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -108,7 +108,7 @@ }, "BLOOM_FILTER_WRONG_TYPE" : { "message" : [ - "Input to function should have been followed by a value with , but it's [, ]." + "Input to function should have been followed by value with , but it's []." ] }, "CANNOT_CONVERT_TO_JSON" : { diff --git a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala index 71419d5aea0b..fe738f414996 100644 --- a/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala +++ b/core/src/main/scala/org/apache/spark/HeartbeatReceiver.scala @@ -25,7 +25,7 @@ import scala.concurrent.Future import org.apache.spark.executor.ExecutorMetrics import org.apache.spark.internal.{config, Logging} import org.apache.spark.internal.config.Network -import org.apache.spark.rpc.{RpcCallContext, RpcEnv, ThreadSafeRpcEndpoint} +import org.apache.spark.rpc.{IsolatedRpcEndpoint, RpcCallContext, RpcEnv} import org.apache.spark.scheduler._ import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.RemoveExecutor import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend @@ -65,7 +65,7 @@ private[spark] case class HeartbeatResponse(reregisterBlockManager: Boolean) * Lives in the driver to receive heartbeats from executors.. */ private[spark] class HeartbeatReceiver(sc: SparkContext, clock: Clock) - extends SparkListener with ThreadSafeRpcEndpoint with Logging { + extends SparkListener with IsolatedRpcEndpoint with Logging { def this(sc: SparkContext) = { this(sc, new SystemClock) diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala index 7b31fa93c32e..14d5df14ed85 100644 --- a/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala +++ b/core/src/main/scala/org/apache/spark/api/python/PythonRunner.scala @@ -403,6 +403,7 @@ private[spark] abstract class BasePythonRunner[IN, OUT]( // the decrypted data to python val idsAndFiles = broadcastVars.flatMap { broadcast => if (!oldBids.contains(broadcast.id)) { + oldBids.add(broadcast.id) Some((broadcast.id, broadcast.value.path)) } else { None @@ -416,7 +417,6 @@ private[spark] abstract class BasePythonRunner[IN, OUT]( idsAndFiles.foreach { case (id, _) => // send new broadcast dataOut.writeLong(id) - oldBids.add(id) } dataOut.flush() logTrace("waiting for python to read decrypted broadcast data from server") diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index b6d6441925a3..4efce34b18c2 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -1883,7 +1883,7 @@ private[spark] class DAGScheduler( if (ignoreStageFailure) { logInfo(s"Ignoring fetch failure from $task of $failedStage attempt " + s"${task.stageAttemptId} when count spark.stage.maxConsecutiveAttempts " + - "as executor ${bmAddress.executorId} is decommissioned and " + + s"as executor ${bmAddress.executorId} is decommissioned and " + s" ${config.STAGE_IGNORE_DECOMMISSION_FETCH_FAILURE.key}=true") } else { failedStage.failedAttemptIds.add(task.stageAttemptId) diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala index 5004262a71c1..1eb588124a7c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala @@ -641,7 +641,7 @@ private[spark] class TaskSchedulerImpl( if (!unschedulableTaskSetToExpiryTime.contains(taskSet)) { logInfo("Notifying ExecutorAllocationManager to allocate more executors to" + " schedule the unschedulable task before aborting" + - " stage ${taskSet.stageId}.") + s" stage ${taskSet.stageId}.") dagScheduler.unschedulableTaskSetAdded(taskSet.taskSet.stageId, taskSet.taskSet.stageAttemptId) updateUnschedulableTaskSetTimeoutAndStartAbortTimer(taskSet, taskIndex) diff --git a/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala b/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala index 8cc24426c4e8..c15ae9504c7b 100644 --- a/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkThrowableSuite.scala @@ -74,7 +74,8 @@ class SparkThrowableSuite extends SparkFunSuite { } test("Error classes are correctly formatted") { - val errorClassFileContents = IOUtils.toString(errorJsonFilePath.toUri.toURL.openStream()) + val errorClassFileContents = + IOUtils.toString(errorJsonFilePath.toUri.toURL.openStream(), StandardCharsets.UTF_8) val mapper = JsonMapper.builder() .addModule(DefaultScalaModule) .enable(SerializationFeature.INDENT_OUTPUT) @@ -101,7 +102,8 @@ class SparkThrowableSuite extends SparkFunSuite { test("SQLSTATE invariants") { val sqlStates = errorReader.errorInfoMap.values.toSeq.flatMap(_.sqlState) val errorClassReadMe = Utils.getSparkClassLoader.getResource("error/README.md") - val errorClassReadMeContents = IOUtils.toString(errorClassReadMe.openStream()) + val errorClassReadMeContents = + IOUtils.toString(errorClassReadMe.openStream(), StandardCharsets.UTF_8) val sqlStateTableRegex = "(?s)(.+)".r val sqlTable = sqlStateTableRegex.findFirstIn(errorClassReadMeContents).get @@ -331,8 +333,8 @@ class SparkThrowableSuite extends SparkFunSuite { | ] | } |} - |""".stripMargin) - val reader = new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURL)) + |""".stripMargin, StandardCharsets.UTF_8) + val reader = new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURI.toURL)) assert(reader.getErrorMessage("DIVIDE_BY_ZERO", Map.empty) == "abc") } } @@ -349,9 +351,9 @@ class SparkThrowableSuite extends SparkFunSuite { | ] | } |} - |""".stripMargin) + |""".stripMargin, StandardCharsets.UTF_8) val e = intercept[SparkException] { - new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURL)) + new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURI.toURL)) } assert(e.getErrorClass === "INTERNAL_ERROR") assert(e.getMessage.contains("DIVIDE.BY_ZERO")) @@ -375,9 +377,9 @@ class SparkThrowableSuite extends SparkFunSuite { | } | } |} - |""".stripMargin) + |""".stripMargin, StandardCharsets.UTF_8) val e = intercept[SparkException] { - new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURL)) + new ErrorClassesJsonReader(Seq(errorJsonFilePath.toUri.toURL, json.toURI.toURL)) } assert(e.getErrorClass === "INTERNAL_ERROR") assert(e.getMessage.contains("BY.ZERO")) diff --git a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala index a12b7034a6df..a8b1304b76f8 100644 --- a/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/executor/CoarseGrainedExecutorBackendSuite.scala @@ -395,7 +395,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite // Fake tasks with different taskIds. val taskDescriptions = (1 to numTasks).map { - taskId => new TaskDescription(taskId, 2, "1", "TASK ${taskId}", 19, + taskId => new TaskDescription(taskId, 2, "1", s"TASK $taskId", 19, 1, mutable.Map.empty, mutable.Map.empty, mutable.Map.empty, new Properties, 1, Map(GPU -> new ResourceInformation(GPU, Array("0", "1"))), data) } @@ -483,7 +483,7 @@ class CoarseGrainedExecutorBackendSuite extends SparkFunSuite // Fake tasks with different taskIds. val taskDescriptions = (1 to numTasks).map { - taskId => new TaskDescription(taskId, 2, "1", "TASK ${taskId}", 19, + taskId => new TaskDescription(taskId, 2, "1", s"TASK $taskId", 19, 1, mutable.Map.empty, mutable.Map.empty, mutable.Map.empty, new Properties, 1, Map(GPU -> new ResourceInformation(GPU, Array("0", "1"))), data) } diff --git a/dev/check-codegen-python.py b/dev/check-codegen-python.py index 616b31eb4740..bcb2b0341da9 100755 --- a/dev/check-codegen-python.py +++ b/dev/check-codegen-python.py @@ -75,7 +75,9 @@ def check_connect_protos(): else: fail( "Generated files for pyspark-connect are out of sync! " - "Please run ./connector/connect/dev/generate_protos.sh" + "If you have touched files under connector/connect/src/main/protobuf, " + "please run ./connector/connect/dev/generate_protos.sh. " + "If you haven't touched any file above, please rebase your PR against main branch." ) diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index ee9977e25923..1d1061aaadbd 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -147,7 +147,7 @@ jetty-util/6.1.26//jetty-util-6.1.26.jar jetty-util/9.4.49.v20220914//jetty-util-9.4.49.v20220914.jar jetty/6.1.26//jetty-6.1.26.jar jline/2.14.6//jline-2.14.6.jar -joda-time/2.11.2//joda-time-2.11.2.jar +joda-time/2.12.0//joda-time-2.12.0.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar @@ -195,11 +195,11 @@ log4j-slf4j2-impl/2.19.0//log4j-slf4j2-impl-2.19.0.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar mesos/1.4.3/shaded-protobuf/mesos-1.4.3-shaded-protobuf.jar -metrics-core/4.2.10//metrics-core-4.2.10.jar -metrics-graphite/4.2.10//metrics-graphite-4.2.10.jar -metrics-jmx/4.2.10//metrics-jmx-4.2.10.jar -metrics-json/4.2.10//metrics-json-4.2.10.jar -metrics-jvm/4.2.10//metrics-jvm-4.2.10.jar +metrics-core/4.2.12//metrics-core-4.2.12.jar +metrics-graphite/4.2.12//metrics-graphite-4.2.12.jar +metrics-jmx/4.2.12//metrics-jmx-4.2.12.jar +metrics-json/4.2.12//metrics-json-4.2.12.jar +metrics-jvm/4.2.12//metrics-jvm-4.2.12.jar minlog/1.3.0//minlog-1.3.0.jar netty-all/4.1.84.Final//netty-all-4.1.84.Final.jar netty-buffer/4.1.84.Final//netty-buffer-4.1.84.Final.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 766a28503e42..39a0e6170586 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -132,7 +132,7 @@ jettison/1.1//jettison-1.1.jar jetty-util-ajax/9.4.49.v20220914//jetty-util-ajax-9.4.49.v20220914.jar jetty-util/9.4.49.v20220914//jetty-util-9.4.49.v20220914.jar jline/2.14.6//jline-2.14.6.jar -joda-time/2.11.2//joda-time-2.11.2.jar +joda-time/2.12.0//joda-time-2.12.0.jar jodd-core/3.5.2//jodd-core-3.5.2.jar jpam/1.1//jpam-1.1.jar json/1.8//json-1.8.jar @@ -179,11 +179,11 @@ log4j-slf4j2-impl/2.19.0//log4j-slf4j2-impl-2.19.0.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar mesos/1.4.3/shaded-protobuf/mesos-1.4.3-shaded-protobuf.jar -metrics-core/4.2.10//metrics-core-4.2.10.jar -metrics-graphite/4.2.10//metrics-graphite-4.2.10.jar -metrics-jmx/4.2.10//metrics-jmx-4.2.10.jar -metrics-json/4.2.10//metrics-json-4.2.10.jar -metrics-jvm/4.2.10//metrics-jvm-4.2.10.jar +metrics-core/4.2.12//metrics-core-4.2.12.jar +metrics-graphite/4.2.12//metrics-graphite-4.2.12.jar +metrics-jmx/4.2.12//metrics-jmx-4.2.12.jar +metrics-json/4.2.12//metrics-json-4.2.12.jar +metrics-jvm/4.2.12//metrics-jvm-4.2.12.jar minlog/1.3.0//minlog-1.3.0.jar netty-all/4.1.84.Final//netty-all-4.1.84.Final.jar netty-buffer/4.1.84.Final//netty-buffer-4.1.84.Final.jar diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index ccf0c932b0ef..2a70bd3f98f4 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -17,9 +17,9 @@ # Image for building and testing Spark branches. Based on Ubuntu 20.04. # See also in https://hub.docker.com/_/ubuntu -FROM ubuntu:focal-20220801 +FROM ubuntu:focal-20220922 -ENV FULL_REFRESH_DATE 20220706 +ENV FULL_REFRESH_DATE 20221019 ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true diff --git a/dev/lint-scala b/dev/lint-scala index ad2be152cfad..ea3b98464b2b 100755 --- a/dev/lint-scala +++ b/dev/lint-scala @@ -23,10 +23,21 @@ SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)" "$SCRIPT_DIR/scalastyle" "$1" # For Spark Connect, we actively enforce scalafmt and check that the produced diff is empty. -./build/mvn -Pscala-2.12 scalafmt:format -Dscalafmt.skip=false -Dscalafmt.validateOnly=true -Dscalafmt.changedOnly=false -pl connector/connect -if [[ $? -ne 0 ]]; then - echo "The scalafmt check failed on connector/connect." +ERRORS=$(./build/mvn \ + -Pscala-2.12 \ + scalafmt:format \ + -Dscalafmt.skip=false \ + -Dscalafmt.validateOnly=true \ + -Dscalafmt.changedOnly=false \ + -pl connector/connect \ + 2>&1 | grep -e "^Requires formatting" \ +) + +if test ! -z "$ERRORS"; then + echo -e "The scalafmt check failed on connector/connect at following occurrences:\n\n$ERRORS\n" echo "Before submitting your change, please make sure to format your code using the following command:" echo "./build/mvn -Pscala-2.12 scalafmt:format -Dscalafmt.skip=fase -Dscalafmt.validateOnly=false -Dscalafmt.changedOnly=false -pl connector/connect" exit 1 +else + echo -e "Scalafmt checks passed." fi diff --git a/dev/requirements.txt b/dev/requirements.txt index 651fc280627e..fa4b6752f145 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -51,4 +51,4 @@ grpcio==1.48.1 protobuf==4.21.6 # Spark Connect python proto generation plugin (optional) -mypy-protobuf +mypy-protobuf==3.3.0 diff --git a/docs/ss-migration-guide.md b/docs/ss-migration-guide.md index 0ca5b00debcc..57fe3a84e12c 100644 --- a/docs/ss-migration-guide.md +++ b/docs/ss-migration-guide.md @@ -30,6 +30,8 @@ Please refer [Migration Guide: SQL, Datasets and DataFrame](sql-migration-guide. - Since Spark 3.4, `Trigger.Once` is deprecated, and users are encouraged to migrate from `Trigger.Once` to `Trigger.AvailableNow`. Please refer [SPARK-39805](https://issues.apache.org/jira/browse/SPARK-39805) for more details. +- Since Spark 3.4, the default value of configuration for Kafka offset fetching (`spark.sql.streaming.kafka.useDeprecatedOffsetFetching`) is changed from `true` to `false`. The default no longer relies consumer group based scheduling, which affect the required ACL. For further details please see [Structured Streaming Kafka Integration](structured-streaming-kafka-integration.html#offset-fetching). + ## Upgrading from Structured Streaming 3.2 to 3.3 - Since Spark 3.3, all stateful operators require hash partitioning with exact grouping keys. In previous versions, all stateful operators except stream-stream join require loose partitioning criteria which opens the possibility on correctness issue. (See [SPARK-38204](https://issues.apache.org/jira/browse/SPARK-38204) for more details.) To ensure backward compatibility, we retain the old behavior with the checkpoint built from older versions. diff --git a/docs/structured-streaming-kafka-integration.md b/docs/structured-streaming-kafka-integration.md index 6121f19e806b..d7b741a3fe6a 100644 --- a/docs/structured-streaming-kafka-integration.md +++ b/docs/structured-streaming-kafka-integration.md @@ -568,8 +568,9 @@ Timestamp offset options require Kafka 0.10.1.0 or higher. ### Offset fetching In Spark 3.0 and before Spark uses KafkaConsumer for offset fetching which could cause infinite wait in the driver. -In Spark 3.1 a new configuration option added spark.sql.streaming.kafka.useDeprecatedOffsetFetching (default: true) -which could be set to `false` allowing Spark to use new offset fetching mechanism using AdminClient. +In Spark 3.1 a new configuration option added spark.sql.streaming.kafka.useDeprecatedOffsetFetching (default: false) +which allows Spark to use new offset fetching mechanism using AdminClient. (Set this to `true` to use old offset fetching with KafkaConsumer.) + When the new mechanism used the following applies. First of all the new approach supports Kafka brokers `0.11.0.0+`. diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala index 3acb0bc3c321..1836b07cfd85 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/regression/GeneralizedLinearRegressionSuite.scala @@ -1686,7 +1686,7 @@ class GeneralizedLinearRegressionSuite extends MLTest with DefaultReadWriteTest .setFeaturesCol("features") val model = trainer.fit(dataset) val actual = model.summary.aic - assert(actual ~= expected(idx) absTol 1e-4, "Model mismatch: GLM with regParam = $regParam.") + assert(actual ~= expected(idx) absTol 1e-4, s"Model mismatch: GLM with regParam = $regParam.") idx += 1 } } diff --git a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala index b73823647323..83d77a0a791e 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala @@ -629,14 +629,14 @@ object DecisionTreeSuite extends SparkFunSuite { case (Some(aNode), Some(bNode)) => checkEqual(aNode, bNode) case (None, None) => case _ => - fail("Only one instance has leftNode defined. (a.leftNode: ${a.leftNode}," + - " b.leftNode: ${b.leftNode})") + fail(s"Only one instance has leftNode defined. (a.leftNode: ${a.leftNode}," + + s" b.leftNode: ${b.leftNode})") } (a.rightNode, b.rightNode) match { case (Some(aNode: Node), Some(bNode: Node)) => checkEqual(aNode, bNode) case (None, None) => - case _ => fail("Only one instance has rightNode defined. (a.rightNode: ${a.rightNode}, " + - "b.rightNode: ${b.rightNode})") + case _ => fail(s"Only one instance has rightNode defined. (a.rightNode: ${a.rightNode}, " + + s"b.rightNode: ${b.rightNode})") } } } diff --git a/pom.xml b/pom.xml index 65dfcdb22340..d933c1c6f6dd 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ If you changes codahale.metrics.version, you also need to change the link to metrics.dropwizard.io in docs/monitoring.md. --> - 4.2.10 + 4.2.12 1.11.1 1.12.0 @@ -192,7 +192,7 @@ 14.0.1 3.1.7 2.36 - 2.11.2 + 2.12.0 3.5.2 3.0.0 0.12.0 diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index e5a48080e833..cc103e4ab00a 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -716,8 +716,10 @@ object SparkProtobuf { dependencyOverrides += "com.google.protobuf" % "protobuf-java" % protoVersion, - (Compile / PB.targets) := Seq( - PB.gens.java -> (Compile / sourceManaged).value, + (Test / PB.protoSources) += (Test / sourceDirectory).value / "resources", + + (Test / PB.targets) := Seq( + PB.gens.java -> target.value / "generated-test-sources" ), (assembly / test) := { }, diff --git a/python/pyspark/pandas/mlflow.py b/python/pyspark/pandas/mlflow.py index 094215743e2b..469349b37eee 100644 --- a/python/pyspark/pandas/mlflow.py +++ b/python/pyspark/pandas/mlflow.py @@ -159,7 +159,7 @@ def load_model( ... lr = LinearRegression() ... lr.fit(train_x, train_y) ... mlflow.sklearn.log_model(lr, "model") - LinearRegression(...) + LinearRegression... Now that our model is logged using MLflow, we load it back and apply it on a pandas-on-Spark dataframe: diff --git a/python/pyspark/sql/connect/README.md b/python/pyspark/sql/connect/README.md index fa712677c120..9fd4d1a4596c 100644 --- a/python/pyspark/sql/connect/README.md +++ b/python/pyspark/sql/connect/README.md @@ -49,3 +49,8 @@ To use the release version of Spark Connect: ./python/run-tests --testnames 'pyspark.sql.tests.connect.test_connect_basic' ``` +## Generate proto generated files for the Python client +1. Install `buf version 1.8.0`: https://docs.buf.build/installation +2. Run `pip install grpcio==1.48.1 protobuf==4.21.6 mypy-protobuf==3.3.0` +3. Run `./connector/connect/dev/generate_protos.sh` +4. Optional Check `./dev/check-codegen-python.py` diff --git a/python/pyspark/sql/connect/client.py b/python/pyspark/sql/connect/client.py index cf825f88e857..0ae075521c63 100644 --- a/python/pyspark/sql/connect/client.py +++ b/python/pyspark/sql/connect/client.py @@ -163,8 +163,8 @@ def _process_batch(self, b: pb2.Response) -> Optional[pandas.DataFrame]: if b.batch is not None and len(b.batch.data) > 0: with pa.ipc.open_stream(b.batch.data) as rd: return rd.read_pandas() - elif b.csv_batch is not None and len(b.csv_batch.data) > 0: - return pd.read_csv(io.StringIO(b.csv_batch.data), delimiter="|") + elif b.json_batch is not None and len(b.json_batch.data) > 0: + return pd.read_json(io.BytesIO(b.json_batch.data), lines=True) return None def _execute_and_fetch(self, req: pb2.Request) -> typing.Optional[pandas.DataFrame]: diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 8e34419fc116..eabcf433ae9b 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -39,6 +39,8 @@ from pyspark.sql.connect.typing import ColumnOrString, ExpressionOrString from pyspark.sql.connect.client import RemoteSparkSession +ColumnOrName = Union[ColumnRef, str] + class GroupingFrame(object): @@ -111,7 +113,7 @@ def withPlan( new_frame._session = session return new_frame - def select(self, *cols: ColumnRef) -> "DataFrame": + def select(self, *cols: ColumnOrName) -> "DataFrame": return DataFrame.withPlan(plan.Project(self._plan, *cols), session=self._session) def agg(self, exprs: Optional[GroupingFrame.MeasuresType]) -> "DataFrame": @@ -197,10 +199,40 @@ def join(self, other: "DataFrame", on: Any, how: Optional[str] = None) -> "DataF def limit(self, n: int) -> "DataFrame": return DataFrame.withPlan(plan.Limit(child=self._plan, limit=n), session=self._session) + def offset(self, n: int) -> "DataFrame": + return DataFrame.withPlan(plan.Offset(child=self._plan, offset=n), session=self._session) + def sort(self, *cols: "ColumnOrString") -> "DataFrame": """Sort by a specific column""" return DataFrame.withPlan(plan.Sort(self._plan, *cols), session=self._session) + def sample( + self, + fraction: float, + *, + withReplacement: bool = False, + seed: Optional[int] = None, + ) -> "DataFrame": + if not isinstance(fraction, float): + raise TypeError(f"'fraction' must be float, but got {type(fraction).__name__}") + if not isinstance(withReplacement, bool): + raise TypeError( + f"'withReplacement' must be bool, but got {type(withReplacement).__name__}" + ) + if seed is not None and not isinstance(seed, int): + raise TypeError(f"'seed' must be None or int, but got {type(seed).__name__}") + + return DataFrame.withPlan( + plan.Sample( + child=self._plan, + lower_bound=0.0, + upper_bound=fraction, + with_replacement=withReplacement, + seed=seed, + ), + session=self._session, + ) + def show(self, n: int, truncate: Optional[Union[bool, int]], vertical: Optional[bool]) -> None: ... diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index 41a2629db886..297b15994d3b 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -23,6 +23,7 @@ Union, cast, TYPE_CHECKING, + Mapping, ) import pyspark.sql.connect.proto as proto @@ -111,6 +112,46 @@ def _child_repr_(self) -> str: return self._child._repr_html_() if self._child is not None else "" +class DataSource(LogicalPlan): + """A datasource with a format and optional a schema from which Spark reads data""" + + def __init__( + self, + format: str = "", + schema: Optional[str] = None, + options: Optional[Mapping[str, str]] = None, + ) -> None: + super().__init__(None) + self.format = format + self.schema = schema + self.options = options + + def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: + plan = proto.Relation() + if self.format is not None: + plan.read.data_source.format = self.format + if self.schema is not None: + plan.read.data_source.schema = self.schema + if self.options is not None: + for k in self.options.keys(): + v = self.options.get(k) + if v is not None: + plan.read.data_source.options[k] = v + return plan + + def _repr_html_(self) -> str: + return f""" +
    +
  • + DataSource
    + format: {self.format} + schema: {self.schema} + options: {self.options} +
    • +
+ """ + + class Read(LogicalPlan): def __init__(self, table_name: str) -> None: super().__init__(None) @@ -153,10 +194,12 @@ def __init__(self, child: Optional["LogicalPlan"], *columns: "ExpressionOrString self._verify_expressions() def _verify_expressions(self) -> None: - """Ensures that all input arguments are instances of Expression.""" + """Ensures that all input arguments are instances of Expression or String.""" for c in self._raw_columns: - if not isinstance(c, Expression): - raise InputValidationError(f"Only Expressions can be used for projections: '{c}'.") + if not isinstance(c, (Expression, str)): + raise InputValidationError( + f"Only Expressions or String can be used for projections: '{c}'." + ) def withAlias(self, alias: str) -> LogicalPlan: self.alias = alias @@ -164,10 +207,16 @@ def withAlias(self, alias: str) -> LogicalPlan: def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: assert self._child is not None - proj_exprs = [ - c.to_plan(session) if isinstance(c, Expression) else self.unresolved_attr(c) - for c in self._raw_columns - ] + proj_exprs = [] + for c in self._raw_columns: + if isinstance(c, Expression): + proj_exprs.append(c.to_plan(session)) + elif c == "*": + exp = proto.Expression() + exp.unresolved_star.SetInParent() + proj_exprs.append(exp) + else: + proj_exprs.append(self.unresolved_attr(c)) common = proto.RelationCommon() if self.alias is not None: common.alias = self.alias @@ -223,10 +272,9 @@ def _repr_html_(self) -> str: class Limit(LogicalPlan): - def __init__(self, child: Optional["LogicalPlan"], limit: int, offset: int = 0) -> None: + def __init__(self, child: Optional["LogicalPlan"], limit: int) -> None: super().__init__(child) self.limit = limit - self.offset = offset def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: assert self._child is not None @@ -237,7 +285,7 @@ def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: def print(self, indent: int = 0) -> str: c_buf = self._child.print(indent + LogicalPlan.INDENT) if self._child else "" - return f"{' ' * indent}\n{c_buf}" + return f"{' ' * indent}\n{c_buf}" def _repr_html_(self) -> str: return f""" @@ -245,6 +293,33 @@ def _repr_html_(self) -> str:
  • Limit
    Limit: {self.limit}
    + {self._child_repr_()} +
    • + + """ + + +class Offset(LogicalPlan): + def __init__(self, child: Optional["LogicalPlan"], offset: int = 0) -> None: + super().__init__(child) + self.offset = offset + + def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: + assert self._child is not None + plan = proto.Relation() + plan.offset.input.CopyFrom(self._child.plan(session)) + plan.offset.offset = self.offset + return plan + + def print(self, indent: int = 0) -> str: + c_buf = self._child.print(indent + LogicalPlan.INDENT) if self._child else "" + return f"{' ' * indent}\n{c_buf}" + + def _repr_html_(self) -> str: + return f""" +
      +
    • + Limit
      Offset: {self.offset}
      {self._child_repr_()}
      • @@ -310,6 +385,56 @@ def _repr_html_(self) -> str: """ +class Sample(LogicalPlan): + def __init__( + self, + child: Optional["LogicalPlan"], + lower_bound: float, + upper_bound: float, + with_replacement: bool, + seed: Optional[int], + ) -> None: + super().__init__(child) + self.lower_bound = lower_bound + self.upper_bound = upper_bound + self.with_replacement = with_replacement + self.seed = seed + + def plan(self, session: Optional["RemoteSparkSession"]) -> proto.Relation: + assert self._child is not None + plan = proto.Relation() + plan.sample.input.CopyFrom(self._child.plan(session)) + plan.sample.lower_bound = self.lower_bound + plan.sample.upper_bound = self.upper_bound + plan.sample.with_replacement = self.with_replacement + if self.seed is not None: + plan.sample.seed.seed = self.seed + return plan + + def print(self, indent: int = 0) -> str: + c_buf = self._child.print(indent + LogicalPlan.INDENT) if self._child else "" + return ( + f"{' ' * indent}" + f"" + f"\n{c_buf}" + ) + + def _repr_html_(self) -> str: + return f""" +
        +
      • + Sample
        + LowerBound: {self.lower_bound}
        + UpperBound: {self.upper_bound}
        + WithReplacement: {self.with_replacement}
        + Seed: {self.seed}
        + {self._child_repr_()} +
        • +
      + """ + + class Aggregate(LogicalPlan): MeasureType = Tuple["ExpressionOrString", str] MeasuresType = Sequence[MeasureType] diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index 910f9d644e37..8de6565bae86 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -33,7 +33,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x1cspark/connect/commands.proto\x1a\x1dspark/connect/relations.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"\xdb\x01\n\x07Request\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0cuser_context\x18\x02 \x01(\x0b\x32".spark.connect.Request.UserContextR\x0buserContext\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x43\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName"\xc4\x07\n\x08Response\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12:\n\x05\x62\x61tch\x18\x02 \x01(\x0b\x32".spark.connect.Response.ArrowBatchH\x00R\x05\x62\x61tch\x12?\n\tcsv_batch\x18\x03 \x01(\x0b\x32 .spark.connect.Response.CSVBatchH\x00R\x08\x63svBatch\x12\x39\n\x07metrics\x18\x04 \x01(\x0b\x32\x1f.spark.connect.Response.MetricsR\x07metrics\x1a\xaf\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12-\n\x12uncompressed_bytes\x18\x02 \x01(\x03R\x11uncompressedBytes\x12)\n\x10\x63ompressed_bytes\x18\x03 \x01(\x03R\x0f\x63ompressedBytes\x12\x12\n\x04\x64\x61ta\x18\x04 \x01(\x0cR\x04\x64\x61ta\x12\x16\n\x06schema\x18\x05 \x01(\x0cR\x06schema\x1a;\n\x08\x43SVBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\tR\x04\x64\x61ta\x1a\xe4\x03\n\x07Metrics\x12\x46\n\x07metrics\x18\x01 \x03(\x0b\x32,.spark.connect.Response.Metrics.MetricObjectR\x07metrics\x1a\xb6\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12o\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32\x42.spark.connect.Response.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1ap\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x41\n\x05value\x18\x02 \x01(\x0b\x32+.spark.connect.Response.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricTypeB\r\n\x0bresult_type"\x9b\x01\n\x0f\x41nalyzeResponse\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12!\n\x0c\x63olumn_types\x18\x03 \x03(\tR\x0b\x63olumnTypes\x12%\n\x0e\x65xplain_string\x18\x04 \x01(\tR\rexplainString2\xa2\x01\n\x13SparkConnectService\x12\x42\n\x0b\x45xecutePlan\x12\x16.spark.connect.Request\x1a\x17.spark.connect.Response"\x00\x30\x01\x12G\n\x0b\x41nalyzePlan\x12\x16.spark.connect.Request\x1a\x1e.spark.connect.AnalyzeResponse"\x00\x42"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x1cspark/connect/commands.proto\x1a\x1dspark/connect/relations.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"\xdb\x01\n\x07Request\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12\x45\n\x0cuser_context\x18\x02 \x01(\x0b\x32".spark.connect.Request.UserContextR\x0buserContext\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x43\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName"\xc8\x07\n\x08Response\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12:\n\x05\x62\x61tch\x18\x02 \x01(\x0b\x32".spark.connect.Response.ArrowBatchH\x00R\x05\x62\x61tch\x12\x42\n\njson_batch\x18\x03 \x01(\x0b\x32!.spark.connect.Response.JSONBatchH\x00R\tjsonBatch\x12\x39\n\x07metrics\x18\x04 \x01(\x0b\x32\x1f.spark.connect.Response.MetricsR\x07metrics\x1a\xaf\x01\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12-\n\x12uncompressed_bytes\x18\x02 \x01(\x03R\x11uncompressedBytes\x12)\n\x10\x63ompressed_bytes\x18\x03 \x01(\x03R\x0f\x63ompressedBytes\x12\x12\n\x04\x64\x61ta\x18\x04 \x01(\x0cR\x04\x64\x61ta\x12\x16\n\x06schema\x18\x05 \x01(\x0cR\x06schema\x1a<\n\tJSONBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x1a\xe4\x03\n\x07Metrics\x12\x46\n\x07metrics\x18\x01 \x03(\x0b\x32,.spark.connect.Response.Metrics.MetricObjectR\x07metrics\x1a\xb6\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12o\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32\x42.spark.connect.Response.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1ap\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x41\n\x05value\x18\x02 \x01(\x0b\x32+.spark.connect.Response.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricTypeB\r\n\x0bresult_type"\x9b\x01\n\x0f\x41nalyzeResponse\x12\x1b\n\tclient_id\x18\x01 \x01(\tR\x08\x63lientId\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12!\n\x0c\x63olumn_types\x18\x03 \x03(\tR\x0b\x63olumnTypes\x12%\n\x0e\x65xplain_string\x18\x04 \x01(\tR\rexplainString2\xa2\x01\n\x13SparkConnectService\x12\x42\n\x0b\x45xecutePlan\x12\x16.spark.connect.Request\x1a\x17.spark.connect.Response"\x00\x30\x01\x12G\n\x0b\x41nalyzePlan\x12\x16.spark.connect.Request\x1a\x1e.spark.connect.AnalyzeResponse"\x00\x42"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -51,21 +51,21 @@ _REQUEST_USERCONTEXT._serialized_start = 375 _REQUEST_USERCONTEXT._serialized_end = 442 _RESPONSE._serialized_start = 445 - _RESPONSE._serialized_end = 1409 - _RESPONSE_ARROWBATCH._serialized_start = 671 - _RESPONSE_ARROWBATCH._serialized_end = 846 - _RESPONSE_CSVBATCH._serialized_start = 848 - _RESPONSE_CSVBATCH._serialized_end = 907 - _RESPONSE_METRICS._serialized_start = 910 - _RESPONSE_METRICS._serialized_end = 1394 - _RESPONSE_METRICS_METRICOBJECT._serialized_start = 994 - _RESPONSE_METRICS_METRICOBJECT._serialized_end = 1304 - _RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 1192 - _RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 1304 - _RESPONSE_METRICS_METRICVALUE._serialized_start = 1306 - _RESPONSE_METRICS_METRICVALUE._serialized_end = 1394 - _ANALYZERESPONSE._serialized_start = 1412 - _ANALYZERESPONSE._serialized_end = 1567 - _SPARKCONNECTSERVICE._serialized_start = 1570 - _SPARKCONNECTSERVICE._serialized_end = 1732 + _RESPONSE._serialized_end = 1413 + _RESPONSE_ARROWBATCH._serialized_start = 674 + _RESPONSE_ARROWBATCH._serialized_end = 849 + _RESPONSE_JSONBATCH._serialized_start = 851 + _RESPONSE_JSONBATCH._serialized_end = 911 + _RESPONSE_METRICS._serialized_start = 914 + _RESPONSE_METRICS._serialized_end = 1398 + _RESPONSE_METRICS_METRICOBJECT._serialized_start = 998 + _RESPONSE_METRICS_METRICOBJECT._serialized_end = 1308 + _RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_start = 1196 + _RESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY._serialized_end = 1308 + _RESPONSE_METRICS_METRICVALUE._serialized_start = 1310 + _RESPONSE_METRICS_METRICVALUE._serialized_end = 1398 + _ANALYZERESPONSE._serialized_start = 1416 + _ANALYZERESPONSE._serialized_end = 1571 + _SPARKCONNECTSERVICE._serialized_start = 1574 + _SPARKCONNECTSERVICE._serialized_end = 1736 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index ff66e69cfa4e..bcac7d11a808 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -195,18 +195,23 @@ class Response(google.protobuf.message.Message): ], ) -> None: ... - class CSVBatch(google.protobuf.message.Message): + class JSONBatch(google.protobuf.message.Message): + """Message type when the result is returned as JSON. This is essentially a bulk wrapper + for the JSON result of a Spark DataFrame. All rows are returned in the JSON record format + of `{col -> row}`. + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor ROW_COUNT_FIELD_NUMBER: builtins.int DATA_FIELD_NUMBER: builtins.int row_count: builtins.int - data: builtins.str + data: builtins.bytes def __init__( self, *, row_count: builtins.int = ..., - data: builtins.str = ..., + data: builtins.bytes = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["data", b"data", "row_count", b"row_count"] @@ -318,13 +323,13 @@ class Response(google.protobuf.message.Message): CLIENT_ID_FIELD_NUMBER: builtins.int BATCH_FIELD_NUMBER: builtins.int - CSV_BATCH_FIELD_NUMBER: builtins.int + JSON_BATCH_FIELD_NUMBER: builtins.int METRICS_FIELD_NUMBER: builtins.int client_id: builtins.str @property def batch(self) -> global___Response.ArrowBatch: ... @property - def csv_batch(self) -> global___Response.CSVBatch: ... + def json_batch(self) -> global___Response.JSONBatch: ... @property def metrics(self) -> global___Response.Metrics: """Metrics for the query execution. Typically, this field is only present in the last @@ -335,7 +340,7 @@ class Response(google.protobuf.message.Message): *, client_id: builtins.str = ..., batch: global___Response.ArrowBatch | None = ..., - csv_batch: global___Response.CSVBatch | None = ..., + json_batch: global___Response.JSONBatch | None = ..., metrics: global___Response.Metrics | None = ..., ) -> None: ... def HasField( @@ -343,8 +348,8 @@ class Response(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "batch", b"batch", - "csv_batch", - b"csv_batch", + "json_batch", + b"json_batch", "metrics", b"metrics", "result_type", @@ -358,8 +363,8 @@ class Response(google.protobuf.message.Message): b"batch", "client_id", b"client_id", - "csv_batch", - b"csv_batch", + "json_batch", + b"json_batch", "metrics", b"metrics", "result_type", @@ -368,7 +373,7 @@ class Response(google.protobuf.message.Message): ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["result_type", b"result_type"] - ) -> typing_extensions.Literal["batch", "csv_batch"] | None: ... + ) -> typing_extensions.Literal["batch", "json_batch"] | None: ... global___Response = Response diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py index c3b7b7ec2eaf..d9a596fba8c6 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.py +++ b/python/pyspark/sql/connect/proto/relations_pb2.py @@ -32,7 +32,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xcf\x05\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12,\n\x05union\x18\x06 \x01(\x0b\x32\x14.spark.connect.UnionH\x00R\x05union\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"G\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias"\x1b\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query"\x95\x01\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x1a=\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifierB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x9d\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType"\xbb\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06"\xcd\x01\n\x05Union\x12/\n\x06inputs\x18\x01 \x03(\x0b\x32\x17.spark.connect.RelationR\x06inputs\x12=\n\nunion_type\x18\x02 \x01(\x0e\x32\x1e.spark.connect.Union.UnionTypeR\tunionType"T\n\tUnionType\x12\x1a\n\x16UNION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13UNION_TYPE_DISTINCT\x10\x01\x12\x12\n\x0eUNION_TYPE_ALL\x10\x02"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"\xc5\x02\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12Y\n\x12result_expressions\x18\x03 \x03(\x0b\x32*.spark.connect.Aggregate.AggregateFunctionR\x11resultExpressions\x1a`\n\x11\x41ggregateFunction\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\xf6\x03\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\x0bsort_fields\x18\x02 \x03(\x0b\x32\x1d.spark.connect.Sort.SortFieldR\nsortFields\x1a\xbc\x01\n\tSortField\x12\x39\n\nexpression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nexpression\x12?\n\tdirection\x18\x02 \x01(\x0e\x32!.spark.connect.Sort.SortDirectionR\tdirection\x12\x33\n\x05nulls\x18\x03 \x01(\x0e\x32\x1d.spark.connect.Sort.SortNullsR\x05nulls"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"R\n\tSortNulls\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02"]\n\rLocalRelation\x12L\n\nattributes\x18\x01 \x03(\x0b\x32,.spark.connect.Expression.QualifiedAttributeR\nattributes"\xb8\x01\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12)\n\x10with_replacement\x18\x04 \x01(\x08R\x0fwithReplacement\x12\x12\n\x04seed\x18\x05 \x01(\x03R\x04seedB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\x8f\x06\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12,\n\x05union\x18\x06 \x01(\x0b\x32\x14.spark.connect.UnionH\x00R\x05union\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"G\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias"\x1b\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query"\x9a\x03\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x1a=\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x1a\xbf\x01\n\nDataSource\x12\x16\n\x06\x66ormat\x18\x01 \x01(\tR\x06\x66ormat\x12\x16\n\x06schema\x18\x02 \x01(\tR\x06schema\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x9d\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType"\xbb\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06"\xcd\x01\n\x05Union\x12/\n\x06inputs\x18\x01 \x03(\x0b\x32\x17.spark.connect.RelationR\x06inputs\x12=\n\nunion_type\x18\x02 \x01(\x0e\x32\x1e.spark.connect.Union.UnionTypeR\tunionType"T\n\tUnionType\x12\x1a\n\x16UNION_TYPE_UNSPECIFIED\x10\x00\x12\x17\n\x13UNION_TYPE_DISTINCT\x10\x01\x12\x12\n\x0eUNION_TYPE_ALL\x10\x02"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"\xc5\x02\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12Y\n\x12result_expressions\x18\x03 \x03(\x0b\x32*.spark.connect.Aggregate.AggregateFunctionR\x11resultExpressions\x1a`\n\x11\x41ggregateFunction\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\xf6\x03\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\x0bsort_fields\x18\x02 \x03(\x0b\x32\x1d.spark.connect.Sort.SortFieldR\nsortFields\x1a\xbc\x01\n\tSortField\x12\x39\n\nexpression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nexpression\x12?\n\tdirection\x18\x02 \x01(\x0e\x32!.spark.connect.Sort.SortDirectionR\tdirection\x12\x33\n\x05nulls\x18\x03 \x01(\x0e\x32\x1d.spark.connect.Sort.SortNullsR\x05nulls"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"R\n\tSortNulls\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02"\x8e\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12-\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08R\x10\x61llColumnsAsKeys"]\n\rLocalRelation\x12L\n\nattributes\x18\x01 \x03(\x0b\x32,.spark.connect.Expression.QualifiedAttributeR\nattributes"\xf0\x01\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12)\n\x10with_replacement\x18\x04 \x01(\x08R\x0fwithReplacement\x12.\n\x04seed\x18\x05 \x01(\x0b\x32\x1a.spark.connect.Sample.SeedR\x04seed\x1a\x1a\n\x04Seed\x12\x12\n\x04seed\x18\x01 \x01(\x03R\x04seedB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) @@ -41,48 +41,58 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\036org.apache.spark.connect.protoP\001" + _READ_DATASOURCE_OPTIONSENTRY._options = None + _READ_DATASOURCE_OPTIONSENTRY._serialized_options = b"8\001" _RELATION._serialized_start = 82 - _RELATION._serialized_end = 801 - _UNKNOWN._serialized_start = 803 - _UNKNOWN._serialized_end = 812 - _RELATIONCOMMON._serialized_start = 814 - _RELATIONCOMMON._serialized_end = 885 - _SQL._serialized_start = 887 - _SQL._serialized_end = 914 - _READ._serialized_start = 917 - _READ._serialized_end = 1066 - _READ_NAMEDTABLE._serialized_start = 992 - _READ_NAMEDTABLE._serialized_end = 1053 - _PROJECT._serialized_start = 1068 - _PROJECT._serialized_end = 1185 - _FILTER._serialized_start = 1187 - _FILTER._serialized_end = 1299 - _JOIN._serialized_start = 1302 - _JOIN._serialized_end = 1715 - _JOIN_JOINTYPE._serialized_start = 1528 - _JOIN_JOINTYPE._serialized_end = 1715 - _UNION._serialized_start = 1718 - _UNION._serialized_end = 1923 - _UNION_UNIONTYPE._serialized_start = 1839 - _UNION_UNIONTYPE._serialized_end = 1923 - _LIMIT._serialized_start = 1925 - _LIMIT._serialized_end = 2001 - _OFFSET._serialized_start = 2003 - _OFFSET._serialized_end = 2082 - _AGGREGATE._serialized_start = 2085 - _AGGREGATE._serialized_end = 2410 - _AGGREGATE_AGGREGATEFUNCTION._serialized_start = 2314 - _AGGREGATE_AGGREGATEFUNCTION._serialized_end = 2410 - _SORT._serialized_start = 2413 - _SORT._serialized_end = 2915 - _SORT_SORTFIELD._serialized_start = 2533 - _SORT_SORTFIELD._serialized_end = 2721 - _SORT_SORTDIRECTION._serialized_start = 2723 - _SORT_SORTDIRECTION._serialized_end = 2831 - _SORT_SORTNULLS._serialized_start = 2833 - _SORT_SORTNULLS._serialized_end = 2915 - _LOCALRELATION._serialized_start = 2917 - _LOCALRELATION._serialized_end = 3010 - _SAMPLE._serialized_start = 3013 - _SAMPLE._serialized_end = 3197 + _RELATION._serialized_end = 865 + _UNKNOWN._serialized_start = 867 + _UNKNOWN._serialized_end = 876 + _RELATIONCOMMON._serialized_start = 878 + _RELATIONCOMMON._serialized_end = 949 + _SQL._serialized_start = 951 + _SQL._serialized_end = 978 + _READ._serialized_start = 981 + _READ._serialized_end = 1391 + _READ_NAMEDTABLE._serialized_start = 1123 + _READ_NAMEDTABLE._serialized_end = 1184 + _READ_DATASOURCE._serialized_start = 1187 + _READ_DATASOURCE._serialized_end = 1378 + _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 1320 + _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 1378 + _PROJECT._serialized_start = 1393 + _PROJECT._serialized_end = 1510 + _FILTER._serialized_start = 1512 + _FILTER._serialized_end = 1624 + _JOIN._serialized_start = 1627 + _JOIN._serialized_end = 2040 + _JOIN_JOINTYPE._serialized_start = 1853 + _JOIN_JOINTYPE._serialized_end = 2040 + _UNION._serialized_start = 2043 + _UNION._serialized_end = 2248 + _UNION_UNIONTYPE._serialized_start = 2164 + _UNION_UNIONTYPE._serialized_end = 2248 + _LIMIT._serialized_start = 2250 + _LIMIT._serialized_end = 2326 + _OFFSET._serialized_start = 2328 + _OFFSET._serialized_end = 2407 + _AGGREGATE._serialized_start = 2410 + _AGGREGATE._serialized_end = 2735 + _AGGREGATE_AGGREGATEFUNCTION._serialized_start = 2639 + _AGGREGATE_AGGREGATEFUNCTION._serialized_end = 2735 + _SORT._serialized_start = 2738 + _SORT._serialized_end = 3240 + _SORT_SORTFIELD._serialized_start = 2858 + _SORT_SORTFIELD._serialized_end = 3046 + _SORT_SORTDIRECTION._serialized_start = 3048 + _SORT_SORTDIRECTION._serialized_end = 3156 + _SORT_SORTNULLS._serialized_start = 3158 + _SORT_SORTNULLS._serialized_end = 3240 + _DEDUPLICATE._serialized_start = 3243 + _DEDUPLICATE._serialized_end = 3385 + _LOCALRELATION._serialized_start = 3387 + _LOCALRELATION._serialized_end = 3480 + _SAMPLE._serialized_start = 3483 + _SAMPLE._serialized_end = 3723 + _SAMPLE_SEED._serialized_start = 3697 + _SAMPLE_SEED._serialized_end = 3723 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi index 3354fc86f45d..df179df1480e 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.pyi +++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi @@ -72,6 +72,7 @@ class Relation(google.protobuf.message.Message): LOCAL_RELATION_FIELD_NUMBER: builtins.int SAMPLE_FIELD_NUMBER: builtins.int OFFSET_FIELD_NUMBER: builtins.int + DEDUPLICATE_FIELD_NUMBER: builtins.int UNKNOWN_FIELD_NUMBER: builtins.int @property def common(self) -> global___RelationCommon: ... @@ -100,6 +101,8 @@ class Relation(google.protobuf.message.Message): @property def offset(self) -> global___Offset: ... @property + def deduplicate(self) -> global___Deduplicate: ... + @property def unknown(self) -> global___Unknown: ... def __init__( self, @@ -117,6 +120,7 @@ class Relation(google.protobuf.message.Message): local_relation: global___LocalRelation | None = ..., sample: global___Sample | None = ..., offset: global___Offset | None = ..., + deduplicate: global___Deduplicate | None = ..., unknown: global___Unknown | None = ..., ) -> None: ... def HasField( @@ -126,6 +130,8 @@ class Relation(google.protobuf.message.Message): b"aggregate", "common", b"common", + "deduplicate", + b"deduplicate", "filter", b"filter", "join", @@ -161,6 +167,8 @@ class Relation(google.protobuf.message.Message): b"aggregate", "common", b"common", + "deduplicate", + b"deduplicate", "filter", b"filter", "join", @@ -204,6 +212,7 @@ class Relation(google.protobuf.message.Message): "local_relation", "sample", "offset", + "deduplicate", "unknown", ] | None: ... @@ -280,29 +289,79 @@ class Read(google.protobuf.message.Message): field_name: typing_extensions.Literal["unparsed_identifier", b"unparsed_identifier"], ) -> None: ... + class DataSource(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class OptionsEntry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + KEY_FIELD_NUMBER: builtins.int + VALUE_FIELD_NUMBER: builtins.int + key: builtins.str + value: builtins.str + def __init__( + self, + *, + key: builtins.str = ..., + value: builtins.str = ..., + ) -> None: ... + def ClearField( + self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] + ) -> None: ... + + FORMAT_FIELD_NUMBER: builtins.int + SCHEMA_FIELD_NUMBER: builtins.int + OPTIONS_FIELD_NUMBER: builtins.int + format: builtins.str + """Required. Supported formats include: parquet, orc, text, json, parquet, csv, avro.""" + schema: builtins.str + """Optional. If not set, Spark will infer the schema.""" + @property + def options( + self, + ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: + """The key is case insensitive.""" + def __init__( + self, + *, + format: builtins.str = ..., + schema: builtins.str = ..., + options: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "format", b"format", "options", b"options", "schema", b"schema" + ], + ) -> None: ... + NAMED_TABLE_FIELD_NUMBER: builtins.int + DATA_SOURCE_FIELD_NUMBER: builtins.int @property def named_table(self) -> global___Read.NamedTable: ... + @property + def data_source(self) -> global___Read.DataSource: ... def __init__( self, *, named_table: global___Read.NamedTable | None = ..., + data_source: global___Read.DataSource | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ - "named_table", b"named_table", "read_type", b"read_type" + "data_source", b"data_source", "named_table", b"named_table", "read_type", b"read_type" ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "named_table", b"named_table", "read_type", b"read_type" + "data_source", b"data_source", "named_table", b"named_table", "read_type", b"read_type" ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["read_type", b"read_type"] - ) -> typing_extensions.Literal["named_table"] | None: ... + ) -> typing_extensions.Literal["named_table", "data_source"] | None: ... global___Read = Read @@ -709,6 +768,47 @@ class Sort(google.protobuf.message.Message): global___Sort = Sort +class Deduplicate(google.protobuf.message.Message): + """Relation of type [[Deduplicate]] which have duplicate rows removed, could consider either only + the subset of columns or all the columns. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + INPUT_FIELD_NUMBER: builtins.int + COLUMN_NAMES_FIELD_NUMBER: builtins.int + ALL_COLUMNS_AS_KEYS_FIELD_NUMBER: builtins.int + @property + def input(self) -> global___Relation: ... + @property + def column_names( + self, + ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ... + all_columns_as_keys: builtins.bool + def __init__( + self, + *, + input: global___Relation | None = ..., + column_names: collections.abc.Iterable[builtins.str] | None = ..., + all_columns_as_keys: builtins.bool = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["input", b"input"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "all_columns_as_keys", + b"all_columns_as_keys", + "column_names", + b"column_names", + "input", + b"input", + ], + ) -> None: ... + +global___Deduplicate = Deduplicate + class LocalRelation(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -739,6 +839,18 @@ class Sample(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor + class Seed(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SEED_FIELD_NUMBER: builtins.int + seed: builtins.int + def __init__( + self, + *, + seed: builtins.int = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["seed", b"seed"]) -> None: ... + INPUT_FIELD_NUMBER: builtins.int LOWER_BOUND_FIELD_NUMBER: builtins.int UPPER_BOUND_FIELD_NUMBER: builtins.int @@ -749,7 +861,8 @@ class Sample(google.protobuf.message.Message): lower_bound: builtins.float upper_bound: builtins.float with_replacement: builtins.bool - seed: builtins.int + @property + def seed(self) -> global___Sample.Seed: ... def __init__( self, *, @@ -757,10 +870,10 @@ class Sample(google.protobuf.message.Message): lower_bound: builtins.float = ..., upper_bound: builtins.float = ..., with_replacement: builtins.bool = ..., - seed: builtins.int = ..., + seed: global___Sample.Seed | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["input", b"input"] + self, field_name: typing_extensions.Literal["input", b"input", "seed", b"seed"] ) -> builtins.bool: ... def ClearField( self, diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py index 285e78e59ae9..66e48eeab76b 100644 --- a/python/pyspark/sql/connect/readwriter.py +++ b/python/pyspark/sql/connect/readwriter.py @@ -15,8 +15,16 @@ # limitations under the License. # + +from typing import Dict, Optional + +from pyspark.sql.connect.column import PrimitiveType from pyspark.sql.connect.dataframe import DataFrame -from pyspark.sql.connect.plan import Read +from pyspark.sql.connect.plan import Read, DataSource +from pyspark.sql.utils import to_str + + +OptionalPrimitiveType = Optional[PrimitiveType] from typing import TYPE_CHECKING @@ -29,8 +37,114 @@ class DataFrameReader: TODO(SPARK-40539) Achieve parity with PySpark. """ - def __init__(self, client: "RemoteSparkSession") -> None: + def __init__(self, client: "RemoteSparkSession"): self._client = client + self._format = "" + self._schema = "" + self._options: Dict[str, str] = {} + + def format(self, source: str) -> "DataFrameReader": + """ + Specifies the input data source format. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + source : str + string, name of the data source, e.g. 'json', 'parquet'. + + """ + self._format = source + return self + + # TODO(SPARK-40539): support StructType in python client and support schema as StructType. + def schema(self, schema: str) -> "DataFrameReader": + """ + Specifies the input schema. + + Some data sources (e.g. JSON) can infer the input schema automatically from data. + By specifying the schema here, the underlying data source can skip the schema + inference step, and thus speed up data loading. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + schema : str + a DDL-formatted string + (For example ``col0 INT, col1 DOUBLE``). + + """ + self._schema = schema + return self + + def option(self, key: str, value: "OptionalPrimitiveType") -> "DataFrameReader": + """ + Adds an input option for the underlying data source. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + key : str + The key for the option to set. key string is case-insensitive. + value + The value for the option to set. + + """ + self._options[key] = str(value) + return self + + def options(self, **options: "OptionalPrimitiveType") -> "DataFrameReader": + """ + Adds input options for the underlying data source. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + **options : dict + The dictionary of string keys and prmitive-type values. + """ + for k in options: + self.option(k, to_str(options[k])) + return self + + def load( + self, + path: Optional[str] = None, + format: Optional[str] = None, + schema: Optional[str] = None, + **options: "OptionalPrimitiveType", + ) -> "DataFrame": + """ + Loads data from a data source and returns it as a :class:`DataFrame`. + + .. versionadded:: 3.4.0 + + Parameters + ---------- + path : str or list, optional + optional string or a list of string for file-system backed data sources. + format : str, optional + optional string for format of the data source. + schema : str, optional + optional DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). + **options : dict + all other string options + """ + if format is not None: + self.format(format) + if schema is not None: + self.schema(schema) + self.options(**options) + if path is not None: + self.option("path", path) + + plan = DataSource(format=self._format, schema=self._schema, options=self._options) + df = DataFrame.withPlan(plan, self._client) + return df def table(self, tableName: str) -> "DataFrame": df = DataFrame.withPlan(Read(tableName), self._client) diff --git a/python/pyspark/sql/protobuf/functions.py b/python/pyspark/sql/protobuf/functions.py index 9f8b90095dfd..2059d868c7c1 100644 --- a/python/pyspark/sql/protobuf/functions.py +++ b/python/pyspark/sql/protobuf/functions.py @@ -31,8 +31,8 @@ def from_protobuf( data: "ColumnOrName", - descFilePath: str, messageName: str, + descFilePath: str, options: Optional[Dict[str, str]] = None, ) -> Column: """ @@ -48,10 +48,10 @@ def from_protobuf( ---------- data : :class:`~pyspark.sql.Column` or str the binary column. - descFilePath : str - the protobuf descriptor in Message GeneratedMessageV3 format. messageName: str the protobuf message name to look for in descriptor file. + descFilePath : str + the protobuf descriptor in Message GeneratedMessageV3 format. options : dict, optional options to control how the protobuf record is parsed. @@ -80,10 +80,10 @@ def from_protobuf( ... f.flush() ... message_name = 'SimpleMessage' ... proto_df = df.select( - ... to_protobuf(df.value, desc_file_path, message_name).alias("value")) + ... to_protobuf(df.value, message_name, desc_file_path).alias("value")) ... proto_df.show(truncate=False) ... proto_df = proto_df.select( - ... from_protobuf(proto_df.value, desc_file_path, message_name).alias("value")) + ... from_protobuf(proto_df.value, message_name, desc_file_path).alias("value")) ... proto_df.show(truncate=False) +----------------------------------------+ |value | @@ -101,7 +101,7 @@ def from_protobuf( assert sc is not None and sc._jvm is not None try: jc = sc._jvm.org.apache.spark.sql.protobuf.functions.from_protobuf( - _to_java_column(data), descFilePath, messageName, options or {} + _to_java_column(data), messageName, descFilePath, options or {} ) except TypeError as e: if str(e) == "'JavaPackage' object is not callable": @@ -110,7 +110,7 @@ def from_protobuf( return Column(jc) -def to_protobuf(data: "ColumnOrName", descFilePath: str, messageName: str) -> Column: +def to_protobuf(data: "ColumnOrName", messageName: str, descFilePath: str) -> Column: """ Converts a column into binary of protobuf format. @@ -120,10 +120,10 @@ def to_protobuf(data: "ColumnOrName", descFilePath: str, messageName: str) -> Co ---------- data : :class:`~pyspark.sql.Column` or str the data column. - descFilePath : str - the protobuf descriptor in Message GeneratedMessageV3 format. messageName: str the protobuf message name to look for in descriptor file. + descFilePath : str + the protobuf descriptor in Message GeneratedMessageV3 format. Notes ----- @@ -150,7 +150,7 @@ def to_protobuf(data: "ColumnOrName", descFilePath: str, messageName: str) -> Co ... f.flush() ... message_name = 'SimpleMessage' ... proto_df = df.select( - ... to_protobuf(df.value, desc_file_path, message_name).alias("suite")) + ... to_protobuf(df.value, message_name, desc_file_path).alias("suite")) ... proto_df.show(truncate=False) +-------------------------------------------+ |suite | @@ -162,7 +162,7 @@ def to_protobuf(data: "ColumnOrName", descFilePath: str, messageName: str) -> Co assert sc is not None and sc._jvm is not None try: jc = sc._jvm.org.apache.spark.sql.protobuf.functions.to_protobuf( - _to_java_column(data), descFilePath, messageName + _to_java_column(data), messageName, descFilePath ) except TypeError as e: if str(e) == "'JavaPackage' object is not callable": diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py index 1a59e7d596ee..f6988a1d1200 100644 --- a/python/pyspark/sql/tests/connect/test_connect_basic.py +++ b/python/pyspark/sql/tests/connect/test_connect_basic.py @@ -16,6 +16,7 @@ # from typing import Any import unittest +import shutil import tempfile import pandas @@ -24,6 +25,7 @@ from pyspark.sql.connect.client import RemoteSparkSession from pyspark.sql.connect.function_builder import udf from pyspark.sql.connect.functions import lit +from pyspark.sql.dataframe import DataFrame from pyspark.testing.connectutils import should_test_connect, connect_requirement_message from pyspark.testing.utils import ReusedPySparkTestCase @@ -35,6 +37,7 @@ class SparkConnectSQLTestCase(ReusedPySparkTestCase): connect: RemoteSparkSession tbl_name: str + df_text: "DataFrame" @classmethod def setUpClass(cls: Any): @@ -44,7 +47,9 @@ def setUpClass(cls: Any): # Create the new Spark Session cls.spark = SparkSession(cls.sc) cls.testData = [Row(key=i, value=str(i)) for i in range(100)] + cls.testDataStr = [Row(key=str(i)) for i in range(100)] cls.df = cls.sc.parallelize(cls.testData).toDF() + cls.df_text = cls.sc.parallelize(cls.testDataStr).toDF() cls.tbl_name = "test_connect_basic_table_1" @@ -101,6 +106,28 @@ def test_simple_binary_expressions(self): res = pandas.DataFrame(data={"id": [0, 30, 60, 90]}) self.assert_(pd.equals(res), f"{pd.to_string()} != {res.to_string()}") + def test_limit_offset(self): + df = self.connect.read.table(self.tbl_name) + pd = df.limit(10).offset(1).toPandas() + self.assertEqual(9, len(pd.index)) + pd2 = df.offset(98).limit(10).toPandas() + self.assertEqual(2, len(pd2.index)) + + def test_simple_datasource_read(self) -> None: + writeDf = self.df_text + tmpPath = tempfile.mkdtemp() + shutil.rmtree(tmpPath) + writeDf.write.text(tmpPath) + + readDf = self.connect.read.format("text").schema("id STRING").load(path=tmpPath) + expectResult = writeDf.collect() + pandasResult = readDf.toPandas() + if pandasResult is None: + self.assertTrue(False, "Empty pandas dataframe") + else: + actualResult = pandasResult.values.tolist() + self.assertEqual(len(expectResult), len(actualResult)) + if __name__ == "__main__": from pyspark.sql.tests.connect.test_connect_basic import * # noqa: F401 diff --git a/python/pyspark/sql/tests/connect/test_connect_plan_only.py b/python/pyspark/sql/tests/connect/test_connect_plan_only.py index c547000bdcf7..3b609db7a028 100644 --- a/python/pyspark/sql/tests/connect/test_connect_plan_only.py +++ b/python/pyspark/sql/tests/connect/test_connect_plan_only.py @@ -18,6 +18,7 @@ from pyspark.testing.connectutils import PlanOnlyTestFixture import pyspark.sql.connect.proto as proto +from pyspark.sql.connect.readwriter import DataFrameReader from pyspark.sql.connect.function_builder import UserDefinedFunction, udf from pyspark.sql.types import StringType @@ -43,11 +44,51 @@ def test_filter(self): self.assertEqual(plan.root.filter.condition.unresolved_function.parts, [">"]) self.assertEqual(len(plan.root.filter.condition.unresolved_function.arguments), 2) + def test_limit(self): + df = self.connect.readTable(table_name=self.tbl_name) + limit_plan = df.limit(10)._plan.to_proto(self.connect) + self.assertEqual(limit_plan.root.limit.limit, 10) + + def test_offset(self): + df = self.connect.readTable(table_name=self.tbl_name) + offset_plan = df.offset(10)._plan.to_proto(self.connect) + self.assertEqual(offset_plan.root.offset.offset, 10) + + def test_sample(self): + df = self.connect.readTable(table_name=self.tbl_name) + plan = df.filter(df.col_name > 3).sample(fraction=0.3)._plan.to_proto(self.connect) + self.assertEqual(plan.root.sample.lower_bound, 0.0) + self.assertEqual(plan.root.sample.upper_bound, 0.3) + self.assertEqual(plan.root.sample.with_replacement, False) + self.assertEqual(plan.root.sample.HasField("seed"), False) + + plan = ( + df.filter(df.col_name > 3) + .sample(withReplacement=True, fraction=0.4, seed=-1) + ._plan.to_proto(self.connect) + ) + self.assertEqual(plan.root.sample.lower_bound, 0.0) + self.assertEqual(plan.root.sample.upper_bound, 0.4) + self.assertEqual(plan.root.sample.with_replacement, True) + self.assertEqual(plan.root.sample.seed.seed, -1) + def test_relation_alias(self): df = self.connect.readTable(table_name=self.tbl_name) plan = df.alias("table_alias")._plan.to_proto(self.connect) self.assertEqual(plan.root.common.alias, "table_alias") + def test_datasource_read(self): + reader = DataFrameReader(self.connect) + df = reader.load(path="test_path", format="text", schema="id INT", op1="opv", op2="opv2") + plan = df._plan.to_proto(self.connect) + data_source = plan.root.read.data_source + self.assertEqual(data_source.format, "text") + self.assertEqual(data_source.schema, "id INT") + self.assertEqual(len(data_source.options), 3) + self.assertEqual(data_source.options.get("path"), "test_path") + self.assertEqual(data_source.options.get("op1"), "opv") + self.assertEqual(data_source.options.get("op2"), "opv2") + def test_simple_udf(self): u = udf(lambda x: "Martin", StringType()) self.assertIsNotNone(u) diff --git a/python/pyspark/sql/tests/connect/test_connect_select_ops.py b/python/pyspark/sql/tests/connect/test_connect_select_ops.py index 3df9ec9a3bca..e89b4b34ea01 100644 --- a/python/pyspark/sql/tests/connect/test_connect_select_ops.py +++ b/python/pyspark/sql/tests/connect/test_connect_select_ops.py @@ -17,15 +17,19 @@ from pyspark.testing.connectutils import PlanOnlyTestFixture from pyspark.sql.connect import DataFrame from pyspark.sql.connect.functions import col -from pyspark.sql.connect.plan import Read, InputValidationError +from pyspark.sql.connect.plan import Read import pyspark.sql.connect.proto as proto class SparkConnectToProtoSuite(PlanOnlyTestFixture): - def test_select_with_literal(self): + def test_select_with_columns_and_strings(self): df = DataFrame.withPlan(Read("table")) self.assertIsNotNone(df.select(col("name"))._plan.to_proto()) - self.assertRaises(InputValidationError, df.select, "name") + self.assertIsNotNone(df.select("name")) + self.assertIsNotNone(df.select("name", "name2")) + self.assertIsNotNone(df.select(col("name"), col("name2"))) + self.assertIsNotNone(df.select(col("name"), "name2")) + self.assertIsNotNone(df.select("*")) def test_join_with_join_type(self): df_left = DataFrame.withPlan(Read("table")) diff --git a/python/pyspark/tests/test_broadcast.py b/python/pyspark/tests/test_broadcast.py index 8185e812e66b..bc4587ffa645 100644 --- a/python/pyspark/tests/test_broadcast.py +++ b/python/pyspark/tests/test_broadcast.py @@ -26,6 +26,7 @@ from pyspark import SparkConf, SparkContext, Broadcast from pyspark.java_gateway import launch_gateway from pyspark.serializers import ChunkedStream +from pyspark.sql import SparkSession, Row class BroadcastTest(unittest.TestCase): @@ -126,6 +127,19 @@ def test_broadcast_for_error_condition(self): with self.assertRaisesRegex(Py4JJavaError, "RuntimeError.*Broadcast.*unpersisted.*driver"): self.sc.parallelize([1]).map(lambda x: bs.unpersist()).collect() + def test_broadcast_in_udfs_with_encryption(self): + conf = SparkConf() + conf.set("spark.io.encryption.enabled", "true") + conf.setMaster("local-cluster[2,1,1024]") + self.sc = SparkContext(conf=conf) + bar = {"a": "aa", "b": "bb"} + foo = self.sc.broadcast(bar) + spark = SparkSession(self.sc) + spark.udf.register("MYUDF", lambda x: foo.value[x] if x else "") + sel = spark.sql("SELECT MYUDF('a') AS a, MYUDF('b') AS b") + self.assertEqual(sel.collect(), [Row(a="aa", b="bb")]) + spark.stop() + class BroadcastFrameProtocolTest(unittest.TestCase): @classmethod diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile index fc529afb1be8..30338b6f91c7 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile @@ -39,7 +39,7 @@ RUN set -ex && \ ln -sv /bin/bash /bin/sh && \ echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \ chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \ - rm -rf /var/cache/apt/* + rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* COPY jars /opt/spark/jars COPY bin /opt/spark/bin diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17 b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17 index 5a360682071b..194242996ca7 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17 +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile.java17 @@ -39,7 +39,7 @@ RUN set -ex && \ ln -sv /bin/bash /bin/sh && \ echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \ chgrp root /etc/passwd && chmod ug+rw /etc/passwd && \ - rm -rf /var/cache/apt/* + rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* COPY jars /opt/spark/jars COPY bin /opt/spark/bin diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile index 03e421062308..e4d62cf45f5d 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/R/Dockerfile @@ -29,7 +29,7 @@ RUN mkdir ${SPARK_HOME}/R RUN \ apt-get update && \ apt install -y r-base r-base-dev && \ - rm -rf /var/cache/apt/* + rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* COPY R ${SPARK_HOME}/R ENV R_HOME /usr/lib/R diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile index 85d92e39ed27..740aa7f0c43d 100644 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && \ apt install -y python3 python3-pip && \ pip3 install --upgrade pip setuptools && \ # Removed the .cache to save space - rm -rf /root/.cache && rm -rf /var/cache/apt/* + rm -rf /root/.cache && rm -rf /var/cache/apt/* && rm -rf /var/lib/apt/lists/* COPY python/pyspark ${SPARK_HOME}/python/pyspark COPY python/lib ${SPARK_HOME}/python/lib diff --git a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json index e5f9b0cb7a6f..aa9c45675c84 100644 --- a/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json +++ b/sql/catalyst/src/main/resources/gregorian-julian-rebase-micros.json @@ -4,15 +4,15 @@ "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Africa/Accra", - "switches" : [ -62135596748, -59006361548, -55850687948, -52695014348, -46383580748, -43227907148, -40072233548, -33760799948, -30605126348, -27449452748, -21138019148, -17982345548, -14826671948, -12220070348, -12219983948, -12219897548, -12219811148, -12219724748, -12219638348, -12219551948, -12219465548, -12219379148, -12219292748, -2208988800 ], - "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ] + "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], + "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Africa/Addis_Ababa", "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ], "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ] }, { "tz" : "Africa/Algiers", - "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12220071132, -12219984732, -12219898332, -12219811932, -12219725532, -12219639132, -12219552732, -12219466332, -12219379932, -12219293532, -2486679072, -2208988800 ], + "switches" : [ -62135597532, -59006362332, -55850688732, -52695015132, -46383581532, -43227907932, -40072234332, -33760800732, -30605127132, -27449453532, -21138019932, -17982346332, -14826672732, -12220071132, -12219984732, -12219898332, -12219811932, -12219725532, -12219639132, -12219552732, -12219466332, -12219379932, -12219293532, -2486592732, -2208988800 ], "diffs" : [ -175668, -89268, -2868, 83532, 169932, 256332, 342732, 429132, 515532, 601932, 688332, 774732, 861132, 774732, 688332, 601932, 515532, 429132, 342732, 256332, 169932, 83532, -2868, -3039, 0 ] }, { "tz" : "Africa/Asmara", @@ -28,8 +28,8 @@ "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Africa/Bangui", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Banjul", "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], @@ -44,8 +44,8 @@ "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ] }, { "tz" : "Africa/Brazzaville", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Bujumbura", "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ], @@ -80,8 +80,8 @@ "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ] }, { "tz" : "Africa/Douala", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/El_Aaiun", "switches" : [ -62135593632, -59006358432, -55850684832, -52695011232, -46383577632, -43227904032, -40072230432, -33760796832, -30605123232, -27449449632, -21138016032, -17982342432, -14826668832, -12220067232, -12219980832, -12219894432, -12219808032, -12219721632, -12219635232, -12219548832, -12219462432, -12219376032, -12219289632, -2208988800 ], @@ -105,7 +105,7 @@ }, { "tz" : "Africa/Juba", "switches" : [ -62135604388, -59006369188, -55850695588, -52695021988, -46383588388, -43227914788, -40072241188, -33760807588, -30605133988, -27449460388, -21138026788, -17982353188, -14826679588, -12220077988, -12219991588, -12219905188, -12219818788, -12219732388, -12219645988, -12219559588, -12219473188, -12219386788, -12219300388, -2208988800 ], - "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ] + "diffs" : [ -172412, -86012, 388, 86788, 173188, 259588, 345988, 432388, 518788, 605188, 691588, 777988, 864388, 777988, 691588, 605188, 518788, 432388, 345988, 259588, 173188, 86788, 388, 0 ] }, { "tz" : "Africa/Kampala", "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ], @@ -120,24 +120,24 @@ "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ] }, { "tz" : "Africa/Kinshasa", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Lagos", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Libreville", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Lome", "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Africa/Luanda", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Lubumbashi", "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ], @@ -148,8 +148,8 @@ "diffs" : [ -172180, -85780, 620, 87020, 173420, 259820, 346220, 432620, 519020, 605420, 691820, 778220, 864620, 778220, 691820, 605420, 519020, 432620, 346220, 259820, 173420, 87020, 620, 0 ] }, { "tz" : "Africa/Malabo", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Maputo", "switches" : [ -62135604620, -59006369420, -55850695820, -52695022220, -46383588620, -43227915020, -40072241420, -33760807820, -30605134220, -27449460620, -21138027020, -17982353420, -14826679820, -12220078220, -12219991820, -12219905420, -12219819020, -12219732620, -12219646220, -12219559820, -12219473420, -12219387020, -12219300620, -2208988800 ], @@ -180,8 +180,8 @@ "diffs" : [ -172788, -86388, 12, 86412, 172812, 259212, 345612, 432012, 518412, 604812, 691212, 777612, 864012, 777612, 691212, 604812, 518412, 432012, 345612, 259212, 172812, 86412, 12, 0 ] }, { "tz" : "Africa/Niamey", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Nouakchott", "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], @@ -192,8 +192,8 @@ "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Africa/Porto-Novo", - "switches" : [ -62135597616, -59006362416, -55850688816, -52695015216, -46383581616, -43227908016, -40072234416, -33760800816, -30605127216, -27449453616, -21138020016, -17982346416, -14826672816, -12220071216, -12219984816, -12219898416, -12219812016, -12219725616, -12219639216, -12219552816, -12219466416, -12219380016, -12219293616, -2208988800 ], - "diffs" : [ -175584, -89184, -2784, 83616, 170016, 256416, 342816, 429216, 515616, 602016, 688416, 774816, 861216, 774816, 688416, 602016, 515616, 429216, 342816, 256416, 170016, 83616, -2784, 0 ] + "switches" : [ -62135597615, -59006362415, -55850688815, -52695015215, -46383581615, -43227908015, -40072234415, -33760800815, -30605127215, -27449453615, -21138020015, -17982346415, -14826672815, -12220071215, -12219984815, -12219898415, -12219812015, -12219725615, -12219639215, -12219552815, -12219466415, -12219380015, -12219293615, -2208988800 ], + "diffs" : [ -175585, -89185, -2785, 83615, 170015, 256415, 342815, 429215, 515615, 602015, 688415, 774815, 861215, 774815, 688415, 602015, 515615, 429215, 342815, 256415, 170015, 83615, -2785, 0 ] }, { "tz" : "Africa/Sao_Tome", "switches" : [ -62135598416, -59006363216, -55850689616, -52695016016, -46383582416, -43227908816, -40072235216, -33760801616, -30605128016, -27449454416, -21138020816, -17982347216, -14826673616, -12220072016, -12219985616, -12219899216, -12219812816, -12219726416, -12219640016, -12219553616, -12219467216, -12219380816, -12219294416, -2713912016, -2208988800 ], @@ -224,12 +224,12 @@ "diffs" : [ -89976, -3576, 82824, 169224, 255624, 342024, 428424, 514824, 601224, 687624, 774024, 860424, 946824, 860424, 774024, 687624, 601224, 514824, 428424, 342024, 255624, 169224, 82824, -3576, 0 ] }, { "tz" : "America/Anguilla", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Antigua", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Araguaina", "switches" : [ -62135585232, -59006350032, -55850676432, -52695002832, -46383569232, -43227895632, -40072222032, -33760788432, -30605114832, -27449441232, -21138007632, -17982334032, -14826660432, -12220058832, -12219972432, -12219886032, -12219799632, -12219713232, -12219626832, -12219540432, -12219454032, -12219367632, -12219281232, -2208988800 ], @@ -288,16 +288,16 @@ "diffs" : [ -178392, -91992, -5592, 80808, 167208, 253608, 340008, 426408, 512808, 599208, 685608, 772008, 858408, 772008, 685608, 599208, 512808, 426408, 340008, 253608, 167208, 80808, -5592, -4608, 0 ] }, { "tz" : "America/Aruba", - "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ], - "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Asuncion", "switches" : [ -62135582960, -59006347760, -55850674160, -52695000560, -46383566960, -43227893360, -40072219760, -33760786160, -30605112560, -27449438960, -21138005360, -17982331760, -14826658160, -12220056560, -12219970160, -12219883760, -12219797360, -12219710960, -12219624560, -12219538160, -12219451760, -12219365360, -12219278960, -2208988800 ], "diffs" : [ -172240, -85840, 560, 86960, 173360, 259760, 346160, 432560, 518960, 605360, 691760, 778160, 864560, 778160, 691760, 605360, 518960, 432560, 346160, 259760, 173360, 86960, 560, 0 ] }, { "tz" : "America/Atikokan", - "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ], - "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ] + "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ], + "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ] }, { "tz" : "America/Atka", "switches" : [ -62135640802, -59006405602, -55850732002, -52695058402, -46383624802, -43227951202, -40072277602, -33760844002, -30605170402, -27449496802, -21138063202, -17982389602, -14826716002, -12220114402, -12220028002, -12219941602, -12219855202, -12219768802, -12219682402, -12219596002, -12219509602, -12219423202, -12219336802, -3225223727, -2208988800 ], @@ -324,8 +324,8 @@ "diffs" : [ -172368, -85968, 432, 86832, 173232, 259632, 346032, 432432, 518832, 605232, 691632, 778032, 864432, 778032, 691632, 605232, 518832, 432432, 346032, 259632, 173232, 86832, 432, 0 ] }, { "tz" : "America/Blanc-Sablon", - "switches" : [ -62135583092, -59006347892, -55850674292, -52695000692, -46383567092, -43227893492, -40072219892, -33760786292, -30605112692, -27449439092, -21138005492, -17982331892, -14826658292, -12220056692, -12219970292, -12219883892, -12219797492, -12219711092, -12219624692, -12219538292, -12219451892, -12219365492, -12219279092, -2713896692 ], - "diffs" : [ -172108, -85708, 692, 87092, 173492, 259892, 346292, 432692, 519092, 605492, 691892, 778292, 864692, 778292, 691892, 605492, 519092, 432692, 346292, 259892, 173492, 87092, 692, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Boa_Vista", "switches" : [ -62135582240, -59006347040, -55850673440, -52694999840, -46383566240, -43227892640, -40072219040, -33760785440, -30605111840, -27449438240, -21138004640, -17982331040, -14826657440, -12220055840, -12219969440, -12219883040, -12219796640, -12219710240, -12219623840, -12219537440, -12219451040, -12219364640, -12219278240, -2208988800 ], @@ -380,8 +380,8 @@ "diffs" : [ -173060, -86660, -260, 86140, 172540, 258940, 345340, 431740, 518140, 604540, 690940, 777340, 863740, 777340, 690940, 604540, 518140, 431740, 345340, 258940, 172540, 86140, -260, 0 ] }, { "tz" : "America/Coral_Harbour", - "switches" : [ -62135574812, -59006339612, -55850666012, -52694992412, -46383558812, -43227885212, -40072211612, -33760778012, -30605104412, -27449430812, -21137997212, -17982323612, -14826650012, -12220048412, -12219962012, -12219875612, -12219789212, -12219702812, -12219616412, -12219530012, -12219443612, -12219357212, -12219270812, -2366733212, -2208988800 ], - "diffs" : [ -176788, -90388, -3988, 82412, 168812, 255212, 341612, 428012, 514412, 600812, 687212, 773612, 860012, 773612, 687212, 600812, 514412, 428012, 341612, 255212, 168812, 82412, -3988, -3600, 0 ] + "switches" : [ -62135577712, -59006342512, -55850668912, -52694995312, -46383561712, -43227888112, -40072214512, -33760780912, -30605107312, -27449433712, -21138000112, -17982326512, -14826652912, -12220051312, -12219964912, -12219878512, -12219792112, -12219705712, -12219619312, -12219532912, -12219446512, -12219360112, -12219273712, -2524502512, -2208988800 ], + "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, -1176, 0 ] }, { "tz" : "America/Cordoba", "switches" : [ -62135581392, -59006346192, -55850672592, -52694998992, -46383565392, -43227891792, -40072218192, -33760784592, -30605110992, -27449437392, -21138003792, -17982330192, -14826656592, -12220054992, -12219968592, -12219882192, -12219795792, -12219709392, -12219622992, -12219536592, -12219450192, -12219363792, -12219277392, -2208988800 ], @@ -392,16 +392,16 @@ "diffs" : [ -171373, -84973, 1427, 87827, 174227, 260627, 347027, 433427, 519827, 606227, 692627, 779027, 865427, 779027, 692627, 606227, 519827, 433427, 347027, 260627, 174227, 87827, 1427, 0 ] }, { "tz" : "America/Creston", - "switches" : [ -62135568836, -59006333636, -55850660036, -52694986436, -46383552836, -43227879236, -40072205636, -33760772036, -30605098436, -27449424836, -21137991236, -17982317636, -14826644036, -12220042436, -12219956036, -12219869636, -12219783236, -12219696836, -12219610436, -12219524036, -12219437636, -12219351236, -12219264836, -2713882436 ], - "diffs" : [ -175564, -89164, -2764, 83636, 170036, 256436, 342836, 429236, 515636, 602036, 688436, 774836, 861236, 774836, 688436, 602036, 515636, 429236, 342836, 256436, 170036, 83636, -2764, 0 ] + "switches" : [ -62135569902, -59006334702, -55850661102, -52694987502, -46383553902, -43227880302, -40072206702, -33760773102, -30605099502, -27449425902, -21137992302, -17982318702, -14826645102, -12220043502, -12219957102, -12219870702, -12219784302, -12219697902, -12219611502, -12219525102, -12219438702, -12219352302, -12219265902, -2717643600 ], + "diffs" : [ -174498, -88098, -1698, 84702, 171102, 257502, 343902, 430302, 516702, 603102, 689502, 775902, 862302, 775902, 689502, 603102, 516702, 430302, 343902, 257502, 171102, 84702, -1698, 0 ] }, { "tz" : "America/Cuiaba", "switches" : [ -62135583340, -59006348140, -55850674540, -52695000940, -46383567340, -43227893740, -40072220140, -33760786540, -30605112940, -27449439340, -21138005740, -17982332140, -14826658540, -12220056940, -12219970540, -12219884140, -12219797740, -12219711340, -12219624940, -12219538540, -12219452140, -12219365740, -12219279340, -2208988800 ], "diffs" : [ -171860, -85460, 940, 87340, 173740, 260140, 346540, 432940, 519340, 605740, 692140, 778540, 864940, 778540, 692140, 605740, 519340, 432940, 346540, 260140, 173740, 87340, 940, 0 ] }, { "tz" : "America/Curacao", - "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ], - "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Danmarkshavn", "switches" : [ -62135592320, -59006357120, -55850683520, -52695009920, -46383576320, -43227902720, -40072229120, -33760795520, -30605121920, -27449448320, -21138014720, -17982341120, -14826667520, -12220065920, -12219979520, -12219893120, -12219806720, -12219720320, -12219633920, -12219547520, -12219461120, -12219374720, -12219288320, -2208988800 ], @@ -409,7 +409,7 @@ }, { "tz" : "America/Dawson", "switches" : [ -62135563340, -59006328140, -55850654540, -52694980940, -46383547340, -43227873740, -40072200140, -33760766540, -30605092940, -27449419340, -21137985740, -17982312140, -14826638540, -12220036940, -12219950540, -12219864140, -12219777740, -12219691340, -12219604940, -12219518540, -12219432140, -12219345740, -12219259340, -2208988800 ], - "diffs" : [ -177460, -91060, -4660, 81740, 168140, 254540, 340940, 427340, 513740, 600140, 686540, 772940, 859340, 772940, 686540, 600140, 513740, 427340, 340940, 254540, 168140, 81740, -4660, 0 ] + "diffs" : [ -181060, -94660, -8260, 78140, 164540, 250940, 337340, 423740, 510140, 596540, 682940, 769340, 855740, 769340, 682940, 596540, 510140, 423740, 337340, 250940, 164540, 78140, -8260, 0 ] }, { "tz" : "America/Dawson_Creek", "switches" : [ -62135567944, -59006332744, -55850659144, -52694985544, -46383551944, -43227878344, -40072204744, -33760771144, -30605097544, -27449423944, -21137990344, -17982316744, -14826643144, -12220041544, -12219955144, -12219868744, -12219782344, -12219695944, -12219609544, -12219523144, -12219436744, -12219350344, -12219263944, -2713881544, -2208988800 ], @@ -424,8 +424,8 @@ "diffs" : [ -174731, -88331, -1931, 84469, 170869, 257269, 343669, 430069, 516469, 602869, 689269, 775669, 862069, 775669, 689269, 602869, 516469, 430069, 343669, 257269, 170869, 84469, -1931, 0 ] }, { "tz" : "America/Dominica", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Edmonton", "switches" : [ -62135569568, -59006334368, -55850660768, -52694987168, -46383553568, -43227879968, -40072206368, -33760772768, -30605099168, -27449425568, -21137991968, -17982318368, -14826644768, -12220043168, -12219956768, -12219870368, -12219783968, -12219697568, -12219611168, -12219524768, -12219438368, -12219351968, -12219265568, -2208988800 ], @@ -472,12 +472,12 @@ "diffs" : [ -171872, -85472, 928, 87328, 173728, 260128, 346528, 432928, 519328, 605728, 692128, 778528, 864928, 778528, 692128, 605728, 519328, 432928, 346528, 260128, 173728, 87328, 928, -430, 0 ] }, { "tz" : "America/Grenada", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Guadeloupe", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Guatemala", "switches" : [ -62135575076, -59006339876, -55850666276, -52694992676, -46383559076, -43227885476, -40072211876, -33760778276, -30605104676, -27449431076, -21137997476, -17982323876, -14826650276, -12220048676, -12219962276, -12219875876, -12219789476, -12219703076, -12219616676, -12219530276, -12219443876, -12219357476, -12219271076, -2208988800 ], @@ -488,8 +488,8 @@ "diffs" : [ -173960, -87560, -1160, 85240, 171640, 258040, 344440, 430840, 517240, 603640, 690040, 776440, 862840, 776440, 690040, 603640, 517240, 430840, 344440, 258040, 171640, 85240, -1160, -840, 0 ] }, { "tz" : "America/Guyana", - "switches" : [ -62135582840, -59006347640, -55850674040, -52695000440, -46383566840, -43227893240, -40072219640, -33760786040, -30605112440, -27449438840, -21138005240, -17982331640, -14826658040, -12220056440, -12219970040, -12219883640, -12219797240, -12219710840, -12219624440, -12219538040, -12219451640, -12219365240, -12219278840, -2208988800 ], - "diffs" : [ -172360, -85960, 440, 86840, 173240, 259640, 346040, 432440, 518840, 605240, 691640, 778040, 864440, 778040, 691640, 605240, 518840, 432440, 346040, 259640, 173240, 86840, 440, 0 ] + "switches" : [ -62135582841, -59006347641, -55850674041, -52695000441, -46383566841, -43227893241, -40072219641, -33760786041, -30605112441, -27449438841, -21138005241, -17982331641, -14826658041, -12220056441, -12219970041, -12219883641, -12219797241, -12219710841, -12219624441, -12219538041, -12219451641, -12219365241, -12219278841, -2208988800 ], + "diffs" : [ -172359, -85959, 441, 86841, 173241, 259641, 346041, 432441, 518841, 605241, 691641, 778041, 864441, 778041, 691641, 605241, 518841, 432441, 346041, 259641, 173241, 86841, 441, 0 ] }, { "tz" : "America/Halifax", "switches" : [ -62135581536, -59006346336, -55850672736, -52694999136, -46383565536, -43227891936, -40072218336, -33760784736, -30605111136, -27449437536, -21138003936, -17982330336, -14826656736, -12220055136, -12219968736, -12219882336, -12219795936, -12219709536, -12219623136, -12219536736, -12219450336, -12219363936, -12219277536, -2208988800 ], @@ -572,8 +572,8 @@ "diffs" : [ -171990, -85590, 810, 87210, 173610, 260010, 346410, 432810, 519210, 605610, 692010, 778410, 864810, 778410, 692010, 605610, 519210, 432810, 346410, 260010, 173610, 87210, 810, 0 ] }, { "tz" : "America/Kralendijk", - "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ], - "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/La_Paz", "switches" : [ -62135580444, -59006345244, -55850671644, -52694998044, -46383564444, -43227890844, -40072217244, -33760783644, -30605110044, -27449436444, -21138002844, -17982329244, -14826655644, -12220054044, -12219967644, -12219881244, -12219794844, -12219708444, -12219622044, -12219535644, -12219449244, -12219362844, -12219276444, -2208988800 ], @@ -592,8 +592,8 @@ "diffs" : [ -175382, -88982, -2582, 83818, 170218, 256618, 343018, 429418, 515818, 602218, 688618, 775018, 861418, 775018, 688618, 602218, 515818, 429418, 343018, 256618, 170218, 83818, -2582, -3600, 0 ] }, { "tz" : "America/Lower_Princes", - "switches" : [ -62135580253, -59006345053, -55850671453, -52694997853, -46383564253, -43227890653, -40072217053, -33760783453, -30605109853, -27449436253, -21138002653, -17982329053, -14826655453, -12220053853, -12219967453, -12219881053, -12219794653, -12219708253, -12219621853, -12219535453, -12219449053, -12219362653, -12219276253, -2208988800 ], - "diffs" : [ -174947, -88547, -2147, 84253, 170653, 257053, 343453, 429853, 516253, 602653, 689053, 775453, 861853, 775453, 689053, 602653, 516253, 429853, 343453, 257053, 170653, 84253, -2147, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Maceio", "switches" : [ -62135588228, -59006353028, -55850679428, -52695005828, -46383572228, -43227898628, -40072225028, -33760791428, -30605117828, -27449444228, -21138010628, -17982337028, -14826663428, -12220061828, -12219975428, -12219889028, -12219802628, -12219716228, -12219629828, -12219543428, -12219457028, -12219370628, -12219284228, -2208988800 ], @@ -608,8 +608,8 @@ "diffs" : [ -172804, -86404, -4, 86396, 172796, 259196, 345596, 431996, 518396, 604796, 691196, 777596, 863996, 777596, 691196, 604796, 518396, 431996, 345596, 259196, 172796, 86396, -4, 0 ] }, { "tz" : "America/Marigot", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Martinique", "switches" : [ -62135582140, -59006346940, -55850673340, -52694999740, -46383566140, -43227892540, -40072218940, -33760785340, -30605111740, -27449438140, -21138004540, -17982330940, -14826657340, -12220055740, -12219969340, -12219882940, -12219796540, -12219710140, -12219623740, -12219537340, -12219450940, -12219364540, -12219278140, -2208988800 ], @@ -664,12 +664,12 @@ "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ] }, { "tz" : "America/Montserrat", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Nassau", - "switches" : [ -62135578230, -59006343030, -55850669430, -52694995830, -46383562230, -43227888630, -40072215030, -33760781430, -30605107830, -27449434230, -21138000630, -17982327030, -14826653430, -12220051830, -12219965430, -12219879030, -12219792630, -12219706230, -12219619830, -12219533430, -12219447030, -12219360630, -12219274230, -2208988800 ], - "diffs" : [ -173370, -86970, -570, 85830, 172230, 258630, 345030, 431430, 517830, 604230, 690630, 777030, 863430, 777030, 690630, 604230, 517830, 431430, 345030, 258630, 172230, 85830, -570, 0 ] + "switches" : [ -62135577748, -59006342548, -55850668948, -52694995348, -46383561748, -43227888148, -40072214548, -33760780948, -30605107348, -27449433748, -21138000148, -17982326548, -14826652948, -12220051348, -12219964948, -12219878548, -12219792148, -12219705748, -12219619348, -12219532948, -12219446548, -12219360148, -12219273748, -2366736148 ], + "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ] }, { "tz" : "America/New_York", "switches" : [ -62135579038, -59006343838, -55850670238, -52694996638, -46383563038, -43227889438, -40072215838, -33760782238, -30605108638, -27449435038, -21138001438, -17982327838, -14826654238, -12220052638, -12219966238, -12219879838, -12219793438, -12219707038, -12219620638, -12219534238, -12219447838, -12219361438, -12219275038, -2717650800 ], @@ -698,6 +698,10 @@ "tz" : "America/North_Dakota/New_Salem", "switches" : [ -62135572461, -59006337261, -55850663661, -52694990061, -46383556461, -43227882861, -40072209261, -33760775661, -30605102061, -27449428461, -21137994861, -17982321261, -14826647661, -12220046061, -12219959661, -12219873261, -12219786861, -12219700461, -12219614061, -12219527661, -12219441261, -12219354861, -12219268461, -2717643600, -2208988800 ], "diffs" : [ -175539, -89139, -2739, 83661, 170061, 256461, 342861, 429261, 515661, 602061, 688461, 774861, 861261, 774861, 688461, 602061, 515661, 429261, 342861, 256461, 170061, 83661, -2739, -3600, 0 ] +}, { + "tz" : "America/Nuuk", + "switches" : [ -62135584384, -59006349184, -55850675584, -52695001984, -46383568384, -43227894784, -40072221184, -33760787584, -30605113984, -27449440384, -21138006784, -17982333184, -14826659584, -12220057984, -12219971584, -12219885184, -12219798784, -12219712384, -12219625984, -12219539584, -12219453184, -12219366784, -12219280384, -2208988800 ], + "diffs" : [ -174416, -88016, -1616, 84784, 171184, 257584, 343984, 430384, 516784, 603184, 689584, 775984, 862384, 775984, 689584, 603184, 516784, 430384, 343984, 257584, 171184, 84784, -1616, 0 ] }, { "tz" : "America/Ojinaga", "switches" : [ -62135571740, -59006336540, -55850662940, -52694989340, -46383555740, -43227882140, -40072208540, -33760774940, -30605101340, -27449427740, -21137994140, -17982320540, -14826646940, -12220045340, -12219958940, -12219872540, -12219786140, -12219699740, -12219613340, -12219526940, -12219440540, -12219354140, -12219267740, -2208988800 ], @@ -724,8 +728,8 @@ "diffs" : [ -172160, -85760, 640, 87040, 173440, 259840, 346240, 432640, 519040, 605440, 691840, 778240, 864640, 778240, 691840, 605440, 519040, 432640, 346240, 259840, 173440, 87040, 640, 660, 0 ] }, { "tz" : "America/Port_of_Spain", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Porto_Acre", "switches" : [ -62135580528, -59006345328, -55850671728, -52694998128, -46383564528, -43227890928, -40072217328, -33760783728, -30605110128, -27449436528, -21138002928, -17982329328, -14826655728, -12220054128, -12219967728, -12219881328, -12219794928, -12219708528, -12219622128, -12219535728, -12219449328, -12219362928, -12219276528, -2208988800 ], @@ -741,7 +745,7 @@ }, { "tz" : "America/Punta_Arenas", "switches" : [ -62135579780, -59006344580, -55850670980, -52694997380, -46383563780, -43227890180, -40072216580, -33760782980, -30605109380, -27449435780, -21138002180, -17982328580, -14826654980, -12220053380, -12219966980, -12219880580, -12219794180, -12219707780, -12219621380, -12219534980, -12219448580, -12219362180, -12219275780, -2524504580, -2208988800 ], - "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, 771380, 684980, 598580, 512180, 425780, 339380, 252980, 166580, 80180, -6220, -6166, 0 ] + "diffs" : [ -179020, -92620, -6220, 80180, 166580, 252980, 339380, 425780, 512180, 598580, 684980, 771380, 857780, 771380, 684980, 598580, 512180, 425780, 339380, 252980, 166580, 80180, -6220, -6165, 0 ] }, { "tz" : "America/Rainy_River", "switches" : [ -62135574104, -59006338904, -55850665304, -52694991704, -46383558104, -43227884504, -40072210904, -33760777304, -30605103704, -27449430104, -21137996504, -17982322904, -14826649304, -12220047704, -12219961304, -12219874904, -12219788504, -12219702104, -12219615704, -12219529304, -12219442904, -12219356504, -12219270104, -2366732504 ], @@ -780,8 +784,8 @@ "diffs" : [ -175128, -88728, -2328, 84072, 170472, 256872, 343272, 429672, 516072, 602472, 688872, 775272, 861672, 775272, 688872, 602472, 516072, 429672, 343272, 256872, 170472, 84072, -2328, 0 ] }, { "tz" : "America/Santiago", - "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ], - "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ] + "switches" : [ -62135579835, -59006344635, -55850671035, -52694997435, -46383563835, -43227890235, -40072216635, -33760783035, -30605109435, -27449435835, -21138002235, -17982328635, -14826655035, -12220053435, -12219967035, -12219880635, -12219794235, -12219707835, -12219621435, -12219535035, -12219448635, -12219362235, -12219275835, -2208988800 ], + "diffs" : [ -175365, -88965, -2565, 83835, 170235, 256635, 343035, 429435, 515835, 602235, 688635, 775035, 861435, 775035, 688635, 602235, 515835, 429435, 343035, 256635, 170235, 83835, -2565, 0 ] }, { "tz" : "America/Santo_Domingo", "switches" : [ -62135580024, -59006344824, -55850671224, -52694997624, -46383564024, -43227890424, -40072216824, -33760783224, -30605109624, -27449436024, -21138002424, -17982328824, -14826655224, -12220053624, -12219967224, -12219880824, -12219794424, -12219708024, -12219621624, -12219535224, -12219448824, -12219362424, -12219276024, -2524504824, -2208988800 ], @@ -804,28 +808,28 @@ "diffs" : [ -86473, -73, 86327, 172727, 259127, 345527, 431927, 518327, 604727, 691127, 777527, 863927, 950327, 863927, 777527, 691127, 604727, 518327, 431927, 345527, 259127, 172727, 86327, -73, 0 ] }, { "tz" : "America/St_Barthelemy", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/St_Johns", "switches" : [ -62135584148, -59006348948, -55850675348, -52695001748, -46383568148, -43227894548, -40072220948, -33760787348, -30605113748, -27449440148, -21138006548, -17982332948, -14826659348, -12220057748, -12219971348, -12219884948, -12219798548, -12219712148, -12219625748, -12219539348, -12219452948, -12219366548, -12219280148, -2208988800 ], "diffs" : [ -172852, -86452, -52, 86348, 172748, 259148, 345548, 431948, 518348, 604748, 691148, 777548, 863948, 777548, 691148, 604748, 518348, 431948, 345548, 259148, 172748, 86348, -52, 0 ] }, { "tz" : "America/St_Kitts", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/St_Lucia", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/St_Thomas", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/St_Vincent", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Swift_Current", "switches" : [ -62135570920, -59006335720, -55850662120, -52694988520, -46383554920, -43227881320, -40072207720, -33760774120, -30605100520, -27449426920, -21137993320, -17982319720, -14826646120, -12220044520, -12219958120, -12219871720, -12219785320, -12219698920, -12219612520, -12219526120, -12219439720, -12219353320, -12219266920, -2208988800 ], @@ -852,20 +856,20 @@ "diffs" : [ -173852, -87452, -1052, 85348, 171748, 258148, 344548, 430948, 517348, 603748, 690148, 776548, 862948, 776548, 690148, 603748, 517348, 430948, 344548, 258148, 171748, 85348, -1052, 0 ] }, { "tz" : "America/Tortola", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Vancouver", "switches" : [ -62135567252, -59006332052, -55850658452, -52694984852, -46383551252, -43227877652, -40072204052, -33760770452, -30605096852, -27449423252, -21137989652, -17982316052, -14826642452, -12220040852, -12219954452, -12219868052, -12219781652, -12219695252, -12219608852, -12219522452, -12219436052, -12219349652, -12219263252, -2713880852 ], "diffs" : [ -173548, -87148, -748, 85652, 172052, 258452, 344852, 431252, 517652, 604052, 690452, 776852, 863252, 776852, 690452, 604052, 517652, 431252, 344852, 258452, 172052, 85652, -748, 0 ] }, { "tz" : "America/Virgin", - "switches" : [ -62135582036, -59006346836, -55850673236, -52694999636, -46383566036, -43227892436, -40072218836, -33760785236, -30605111636, -27449438036, -21138004436, -17982330836, -14826657236, -12220055636, -12219969236, -12219882836, -12219796436, -12219710036, -12219623636, -12219537236, -12219450836, -12219364436, -12219278036, -2208988800 ], - "diffs" : [ -173164, -86764, -364, 86036, 172436, 258836, 345236, 431636, 518036, 604436, 690836, 777236, 863636, 777236, 690836, 604436, 518036, 431636, 345236, 258836, 172436, 86036, -364, 0 ] + "switches" : [ -62135580935, -59006345735, -55850672135, -52694998535, -46383564935, -43227891335, -40072217735, -33760784135, -30605110535, -27449436935, -21138003335, -17982329735, -14826656135, -12220054535, -12219968135, -12219881735, -12219795335, -12219708935, -12219622535, -12219536135, -12219449735, -12219363335, -12219276935, -2233035335 ], + "diffs" : [ -174265, -87865, -1465, 84935, 171335, 257735, 344135, 430535, 516935, 603335, 689735, 776135, 862535, 776135, 689735, 603335, 516935, 430535, 344135, 257735, 171335, 84935, -1465, 0 ] }, { "tz" : "America/Whitehorse", "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ], - "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ] + "diffs" : [ -180012, -93612, -7212, 79188, 165588, 251988, 338388, 424788, 511188, 597588, 683988, 770388, 856788, 770388, 683988, 597588, 511188, 424788, 338388, 251988, 165588, 79188, -7212, 0 ] }, { "tz" : "America/Winnipeg", "switches" : [ -62135573484, -59006338284, -55850664684, -52694991084, -46383557484, -43227883884, -40072210284, -33760776684, -30605103084, -27449429484, -21137995884, -17982322284, -14826648684, -12220047084, -12219960684, -12219874284, -12219787884, -12219701484, -12219615084, -12219528684, -12219442284, -12219355884, -12219269484, -2602258284 ], @@ -881,19 +885,19 @@ }, { "tz" : "Antarctica/Casey", "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], - "diffs" : [ -201600, -115200, -28800, 57600, 144000, 230400, 316800, 403200, 489600, 576000, 662400, 748800, 835200, 748800, 662400, 576000, 489600, 403200, 316800, 230400, 144000, 57600, -28800, 0 ] + "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, 738000, 651600, 565200, 478800, 392400, 306000, 219600, 133200, 46800, -39600, 0 ] }, { "tz" : "Antarctica/Davis", "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], "diffs" : [ -198000, -111600, -25200, 61200, 147600, 234000, 320400, 406800, 493200, 579600, 666000, 752400, 838800, 752400, 666000, 579600, 493200, 406800, 320400, 234000, 147600, 61200, -25200, 0 ] }, { "tz" : "Antarctica/DumontDUrville", - "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], - "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, 741600, 655200, 568800, 482400, 396000, 309600, 223200, 136800, 50400, -36000, 0 ] + "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ], + "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ] }, { "tz" : "Antarctica/Macquarie", - "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2214259200, -2208988800 ], - "diffs" : [ -212400, -126000, -39600, 46800, 133200, 219600, 306000, 392400, 478800, 565200, 651600, 738000, 824400, 738000, 651600, 565200, 478800, 392400, 306000, 219600, 133200, 46800, -39600, -3600, 0 ] + "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2214259200 ], + "diffs" : [ -208800, -122400, -36000, 50400, 136800, 223200, 309600, 396000, 482400, 568800, 655200, 741600, 828000, 741600, 655200, 568800, 482400, 396000, 309600, 223200, 136800, 50400, -36000, 0 ] }, { "tz" : "Antarctica/Mawson", "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], @@ -916,20 +920,20 @@ "diffs" : [ -174056, -87656, -1256, 85144, 171544, 257944, 344344, 430744, 517144, 603544, 689944, 776344, 862744, 776344, 689944, 603544, 517144, 430744, 344344, 257944, 171544, 85144, -1256, -1800, 0 ] }, { "tz" : "Antarctica/Syowa", - "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], - "diffs" : [ -183600, -97200, -10800, 75600, 162000, 248400, 334800, 421200, 507600, 594000, 680400, 766800, 853200, 766800, 680400, 594000, 507600, 421200, 334800, 248400, 162000, 75600, -10800, 0 ] + "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ], + "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ] }, { "tz" : "Antarctica/Troll", "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ], "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ] }, { "tz" : "Antarctica/Vostok", - "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], - "diffs" : [ -194400, -108000, -21600, 64800, 151200, 237600, 324000, 410400, 496800, 583200, 669600, 756000, 842400, 756000, 669600, 583200, 496800, 410400, 324000, 237600, 151200, 64800, -21600, 0 ] + "switches" : [ -62135617820, -59006382620, -55850709020, -52695035420, -46383601820, -43227928220, -40072254620, -33760821020, -30605147420, -27449473820, -21138040220, -17982366620, -14826693020, -12220091420, -12220005020, -12219918620, -12219832220, -12219745820, -12219659420, -12219573020, -12219486620, -12219400220, -12219313820, -2208988800 ], + "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ] }, { "tz" : "Arctic/Longyearbyen", - "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ], - "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ] + "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ], + "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ] }, { "tz" : "Asia/Aden", "switches" : [ -62135608012, -59006372812, -55850699212, -52695025612, -46383592012, -43227918412, -40072244812, -33760811212, -30605137612, -27449464012, -21138030412, -17982356812, -14826683212, -12220081612, -12219995212, -12219908812, -12219822412, -12219736012, -12219649612, -12219563212, -12219476812, -12219390412, -12219304012, -2208988800 ], @@ -996,8 +1000,8 @@ "diffs" : [ -176496, -90096, -3696, 82704, 169104, 255504, 341904, 428304, 514704, 601104, 687504, 773904, 860304, 773904, 687504, 601104, 514704, 428304, 341904, 255504, 169104, 82704, -3696, 0 ] }, { "tz" : "Asia/Brunei", - "switches" : [ -62135624380, -59006389180, -55850715580, -52695041980, -46383608380, -43227934780, -40072261180, -33760827580, -30605153980, -27449480380, -21138046780, -17982373180, -14826699580, -12220097980, -12220011580, -12219925180, -12219838780, -12219752380, -12219665980, -12219579580, -12219493180, -12219406780, -12219320380, -2208988800 ], - "diffs" : [ -174020, -87620, -1220, 85180, 171580, 257980, 344380, 430780, 517180, 603580, 689980, 776380, 862780, 776380, 689980, 603580, 517180, 430780, 344380, 257980, 171580, 85180, -1220, 0 ] + "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12220096880, -12220010480, -12219924080, -12219837680, -12219751280, -12219664880, -12219578480, -12219492080, -12219405680, -12219319280, -2208988800 ], + "diffs" : [ -175120, -88720, -2320, 84080, 170480, 256880, 343280, 429680, 516080, 602480, 688880, 775280, 861680, 775280, 688880, 602480, 516080, 429680, 343280, 256880, 170480, 84080, -2320, 0 ] }, { "tz" : "Asia/Calcutta", "switches" : [ -62135618008, -59006382808, -55850709208, -52695035608, -46383602008, -43227928408, -40072254808, -33760821208, -30605147608, -27449474008, -21138040408, -17982366808, -14826693208, -12220091608, -12220005208, -12219918808, -12219832408, -12219746008, -12219659608, -12219573208, -12219486808, -12219400408, -12219314008, -3645237208, -3155694800, -2208988800 ], @@ -1064,8 +1068,8 @@ "diffs" : [ -171577, -85177, 1223, 87623, 174023, 260423, 346823, 433223, 519623, 606023, 692423, 778823, 865223, 778823, 692423, 606023, 519623, 433223, 346823, 260423, 174023, 87623, 1223, 0 ] }, { "tz" : "Asia/Ho_Chi_Minh", - "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ], - "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ] + "switches" : [ -62135622390, -59006387190, -55850713590, -52695039990, -46383606390, -43227932790, -40072259190, -33760825590, -30605151990, -27449478390, -21138044790, -17982371190, -14826697590, -12220095990, -12220009590, -12219923190, -12219836790, -12219750390, -12219663990, -12219577590, -12219491190, -12219404790, -12219318390, -2208988800 ], + "diffs" : [ -172410, -86010, 390, 86790, 173190, 259590, 345990, 432390, 518790, 605190, 691590, 777990, 864390, 777990, 691590, 605190, 518790, 432390, 345990, 259590, 173190, 86790, 390, 0 ] }, { "tz" : "Asia/Hong_Kong", "switches" : [ -62135624202, -59006389002, -55850715402, -52695041802, -46383608202, -43227934602, -40072261002, -33760827402, -30605153802, -27449480202, -21138046602, -17982373002, -14826699402, -12220097802, -12220011402, -12219925002, -12219838602, -12219752202, -12219665802, -12219579402, -12219493002, -12219406602, -12219320202, -2208988800 ], @@ -1132,8 +1136,8 @@ "diffs" : [ -175714, -89314, -2914, 83486, 169886, 256286, 342686, 429086, 515486, 601886, 688286, 774686, 861086, 774686, 688286, 601886, 515486, 429086, 342686, 256286, 169886, 83486, -2914, 0 ] }, { "tz" : "Asia/Kuala_Lumpur", - "switches" : [ -62135621206, -59006386006, -55850712406, -52695038806, -46383605206, -43227931606, -40072258006, -33760824406, -30605150806, -27449477206, -21138043606, -17982370006, -14826696406, -12220094806, -12220008406, -12219922006, -12219835606, -12219749206, -12219662806, -12219576406, -12219490006, -12219403606, -12219317206, -2208988800 ], - "diffs" : [ -177194, -90794, -4394, 82006, 168406, 254806, 341206, 427606, 514006, 600406, 686806, 773206, 859606, 773206, 686806, 600406, 514006, 427606, 341206, 254806, 168406, 82006, -4394, 0 ] + "switches" : [ -62135621725, -59006386525, -55850712925, -52695039325, -46383605725, -43227932125, -40072258525, -33760824925, -30605151325, -27449477725, -21138044125, -17982370525, -14826696925, -12220095325, -12220008925, -12219922525, -12219836125, -12219749725, -12219663325, -12219576925, -12219490525, -12219404125, -12219317725, -2208988800 ], + "diffs" : [ -176675, -90275, -3875, 82525, 168925, 255325, 341725, 428125, 514525, 600925, 687325, 773725, 860125, 773725, 687325, 600925, 514525, 428125, 341725, 255325, 168925, 82525, -3875, 0 ] }, { "tz" : "Asia/Kuching", "switches" : [ -62135623280, -59006388080, -55850714480, -52695040880, -46383607280, -43227933680, -40072260080, -33760826480, -30605152880, -27449479280, -21138045680, -17982372080, -14826698480, -12220096880, -12220010480, -12219924080, -12219837680, -12219751280, -12219664880, -12219578480, -12219492080, -12219405680, -12219319280, -2208988800 ], @@ -1220,8 +1224,8 @@ "diffs" : [ -172388, -85988, 412, 86812, 173212, 259612, 346012, 432412, 518812, 605212, 691612, 778012, 864412, 778012, 691612, 605212, 518812, 432412, 346012, 259612, 173212, 86812, 412, 0 ] }, { "tz" : "Asia/Saigon", - "switches" : [ -62135622400, -59006387200, -55850713600, -52695040000, -46383606400, -43227932800, -40072259200, -33760825600, -30605152000, -27449478400, -21138044800, -17982371200, -14826697600, -12220096000, -12220009600, -12219923200, -12219836800, -12219750400, -12219664000, -12219577600, -12219491200, -12219404800, -12219318400, -2208988800 ], - "diffs" : [ -172400, -86000, 400, 86800, 173200, 259600, 346000, 432400, 518800, 605200, 691600, 778000, 864400, 778000, 691600, 605200, 518800, 432400, 346000, 259600, 173200, 86800, 400, 0 ] + "switches" : [ -62135622390, -59006387190, -55850713590, -52695039990, -46383606390, -43227932790, -40072259190, -33760825590, -30605151990, -27449478390, -21138044790, -17982371190, -14826697590, -12220095990, -12220009590, -12219923190, -12219836790, -12219750390, -12219663990, -12219577590, -12219491190, -12219404790, -12219318390, -2208988800 ], + "diffs" : [ -172410, -86010, 390, 86790, 173190, 259590, 345990, 432390, 518790, 605190, 691590, 777990, 864390, 777990, 691590, 605190, 518790, 432390, 345990, 259590, 173190, 86790, 390, 0 ] }, { "tz" : "Asia/Sakhalin", "switches" : [ -62135631048, -59006395848, -55850722248, -52695048648, -46383615048, -43227941448, -40072267848, -33760834248, -30605160648, -27449487048, -21138053448, -17982379848, -14826706248, -12220104648, -12220018248, -12219931848, -12219845448, -12219759048, -12219672648, -12219586248, -12219499848, -12219413448, -12219327048, -2208988800 ], @@ -1352,16 +1356,16 @@ "diffs" : [ -174424, -88024, -1624, 84776, 171176, 257576, 343976, 430376, 516776, 603176, 689576, 775976, 862376, 775976, 689576, 603176, 516776, 430376, 343976, 257576, 171176, 84776, -1624, 0 ] }, { "tz" : "Atlantic/Jan_Mayen", - "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ], - "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ] + "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ], + "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ] }, { "tz" : "Atlantic/Madeira", "switches" : [ -62135592744, -59006357544, -55850683944, -52695010344, -46383576744, -43227903144, -40072229544, -33760795944, -30605122344, -27449448744, -21138015144, -17982341544, -14826667944, -12220066344, -12219979944, -12219893544, -12219807144, -12219720744, -12219634344, -12219547944, -12219461544, -12219375144, -12219288744, -2208988800 ], "diffs" : [ -176856, -90456, -4056, 82344, 168744, 255144, 341544, 427944, 514344, 600744, 687144, 773544, 859944, 773544, 687144, 600744, 514344, 427944, 341544, 255144, 168744, 82344, -4056, 0 ] }, { "tz" : "Atlantic/Reykjavik", - "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ], - "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ] + "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], + "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Atlantic/South_Georgia", "switches" : [ -62135588032, -59006352832, -55850679232, -52695005632, -46383572032, -43227898432, -40072224832, -33760791232, -30605117632, -27449444032, -21138010432, -17982336832, -14826663232, -12220061632, -12219975232, -12219888832, -12219802432, -12219716032, -12219629632, -12219543232, -12219456832, -12219370432, -12219284032, -2524512832 ], @@ -1396,8 +1400,8 @@ "diffs" : [ -172508, -86108, 292, 86692, 173092, 259492, 345892, 432292, 518692, 605092, 691492, 777892, 864292, 777892, 691492, 605092, 518692, 432292, 345892, 259492, 173092, 86692, 292, 0 ] }, { "tz" : "Australia/Currie", - "switches" : [ -62135631328, -59006396128, -55850722528, -52695048928, -46383615328, -43227941728, -40072268128, -33760834528, -30605160928, -27449487328, -21138053728, -17982380128, -14826706528, -12220104928, -12220018528, -12219932128, -12219845728, -12219759328, -12219672928, -12219586528, -12219500128, -12219413728, -12219327328, -2345794528 ], - "diffs" : [ -174272, -87872, -1472, 84928, 171328, 257728, 344128, 430528, 516928, 603328, 689728, 776128, 862528, 776128, 689728, 603328, 516928, 430528, 344128, 257728, 171328, 84928, -1472, 0 ] + "switches" : [ -62135632156, -59006396956, -55850723356, -52695049756, -46383616156, -43227942556, -40072268956, -33760835356, -30605161756, -27449488156, -21138054556, -17982380956, -14826707356, -12220105756, -12220019356, -12219932956, -12219846556, -12219760156, -12219673756, -12219587356, -12219500956, -12219414556, -12219328156, -2345795356 ], + "diffs" : [ -173444, -87044, -644, 85756, 172156, 258556, 344956, 431356, 517756, 604156, 690556, 776956, 863356, 776956, 690556, 604156, 517756, 431356, 344956, 258556, 172156, 85756, -644, 0 ] }, { "tz" : "Australia/Darwin", "switches" : [ -62135628200, -59006393000, -55850719400, -52695045800, -46383612200, -43227938600, -40072265000, -33760831400, -30605157800, -27449484200, -21138050600, -17982377000, -14826703400, -12220101800, -12220015400, -12219929000, -12219842600, -12219756200, -12219669800, -12219583400, -12219497000, -12219410600, -12219324200, -2364108200, -2230189200 ], @@ -1521,11 +1525,11 @@ }, { "tz" : "Canada/Yukon", "switches" : [ -62135564388, -59006329188, -55850655588, -52694981988, -46383548388, -43227874788, -40072201188, -33760767588, -30605093988, -27449420388, -21137986788, -17982313188, -14826639588, -12220037988, -12219951588, -12219865188, -12219778788, -12219692388, -12219605988, -12219519588, -12219433188, -12219346788, -12219260388, -2208988800 ], - "diffs" : [ -176412, -90012, -3612, 82788, 169188, 255588, 341988, 428388, 514788, 601188, 687588, 773988, 860388, 773988, 687588, 601188, 514788, 428388, 341988, 255588, 169188, 82788, -3612, 0 ] + "diffs" : [ -180012, -93612, -7212, 79188, 165588, 251988, 338388, 424788, 511188, 597588, 683988, 770388, 856788, 770388, 683988, 597588, 511188, 424788, 338388, 251988, 165588, 79188, -7212, 0 ] }, { "tz" : "Chile/Continental", - "switches" : [ -62135579834, -59006344634, -55850671034, -52694997434, -46383563834, -43227890234, -40072216634, -33760783034, -30605109434, -27449435834, -21138002234, -17982328634, -14826655034, -12220053434, -12219967034, -12219880634, -12219794234, -12219707834, -12219621434, -12219535034, -12219448634, -12219362234, -12219275834, -2208988800 ], - "diffs" : [ -175366, -88966, -2566, 83834, 170234, 256634, 343034, 429434, 515834, 602234, 688634, 775034, 861434, 775034, 688634, 602234, 515834, 429434, 343034, 256634, 170234, 83834, -2566, 0 ] + "switches" : [ -62135579835, -59006344635, -55850671035, -52694997435, -46383563835, -43227890235, -40072216635, -33760783035, -30605109435, -27449435835, -21138002235, -17982328635, -14826655035, -12220053435, -12219967035, -12219880635, -12219794235, -12219707835, -12219621435, -12219535035, -12219448635, -12219362235, -12219275835, -2208988800 ], + "diffs" : [ -175365, -88965, -2565, 83835, 170235, 256635, 343035, 429435, 515835, 602235, 688635, 775035, 861435, 775035, 688635, 602235, 515835, 429435, 343035, 256635, 170235, 83835, -2565, 0 ] }, { "tz" : "Chile/EasterIsland", "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ], @@ -1548,8 +1552,8 @@ "diffs" : [ -172491, -86091, 309, 86709, 173109, 259509, 345909, 432309, 518709, 605109, 691509, 777909, 864309, 777909, 691509, 605109, 518709, 432309, 345909, 259509, 173109, 86709, 309, 0 ] }, { "tz" : "Eire", - "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ], - "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ] + "switches" : [ -62135595279, -59006360079, -55850686479, -52695012879, -46383579279, -43227905679, -40072232079, -33760798479, -30605124879, -27449451279, -21138017679, -17982344079, -14826670479, -12220068879, -12219982479, -12219896079, -12219809679, -12219723279, -12219636879, -12219550479, -12219464079, -12219377679, -12219291279, -2208988800 ], + "diffs" : [ -174321, -87921, -1521, 84879, 171279, 257679, 344079, 430479, 516879, 603279, 689679, 776079, 862479, 776079, 689679, 603279, 516879, 430479, 344079, 257679, 171279, 84879, -1521, 0 ] }, { "tz" : "Etc/GMT", "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800 ], @@ -1692,8 +1696,8 @@ "diffs" : [ -172800, -86400, 0, 86400, 172800, 259200, 345600, 432000, 518400, 604800, 691200, 777600, 864000, 777600, 691200, 604800, 518400, 432000, 345600, 259200, 172800, 86400, 0 ] }, { "tz" : "Europe/Amsterdam", - "switches" : [ -62135597972, -59006362772, -55850689172, -52695015572, -46383581972, -43227908372, -40072234772, -33760801172, -30605127572, -27449453972, -21138020372, -17982346772, -14826673172, -12220071572, -12219985172, -12219898772, -12219812372, -12219725972, -12219639572, -12219553172, -12219466772, -12219380372, -12219293972, -2208988800 ], - "diffs" : [ -175228, -88828, -2428, 83972, 170372, 256772, 343172, 429572, 515972, 602372, 688772, 775172, 861572, 775172, 688772, 602372, 515972, 429572, 343172, 256772, 170372, 83972, -2428, 0 ] + "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12220071450, -12219985050, -12219898650, -12219812250, -12219725850, -12219639450, -12219553050, -12219466650, -12219380250, -12219293850, -2450995200, -2208988800 ], + "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, 775050, 688650, 602250, 515850, 429450, 343050, 256650, 170250, 83850, -2550, -3600, 0 ] }, { "tz" : "Europe/Andorra", "switches" : [ -62135597164, -59006361964, -55850688364, -52695014764, -46383581164, -43227907564, -40072233964, -33760800364, -30605126764, -27449453164, -21138019564, -17982345964, -14826672364, -12220070764, -12219984364, -12219897964, -12219811564, -12219725164, -12219638764, -12219552364, -12219465964, -12219379564, -12219293164, -2208988800 ], @@ -1732,7 +1736,7 @@ "diffs" : [ -173736, -87336, -936, 85464, 171864, 258264, 344664, 431064, 517464, 603864, 690264, 776664, 863064, 776664, 690264, 603864, 517464, 431064, 344664, 258264, 171864, 85464, -936, 0 ] }, { "tz" : "Europe/Budapest", - "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12220074980, -12219988580, -12219902180, -12219815780, -12219729380, -12219642980, -12219556580, -12219470180, -12219383780, -12219297380, -2500938980 ], + "switches" : [ -62135601380, -59006366180, -55850692580, -52695018980, -46383585380, -43227911780, -40072238180, -33760804580, -30605130980, -27449457380, -21138023780, -17982350180, -14826676580, -12220074980, -12219988580, -12219902180, -12219815780, -12219729380, -12219642980, -12219556580, -12219470180, -12219383780, -12219297380, -2498260580 ], "diffs" : [ -171820, -85420, 980, 87380, 173780, 260180, 346580, 432980, 519380, 605780, 692180, 778580, 864980, 778580, 692180, 605780, 519380, 432980, 346580, 260180, 173780, 87380, 980, 0 ] }, { "tz" : "Europe/Busingen", @@ -1744,12 +1748,12 @@ "diffs" : [ -173080, -86680, -280, 86120, 172520, 258920, 345320, 431720, 518120, 604520, 690920, 777320, 863720, 777320, 690920, 604520, 518120, 431720, 345320, 258920, 172520, 86120, -280, -300, 0 ] }, { "tz" : "Europe/Copenhagen", - "switches" : [ -62135599820, -59006364620, -55850691020, -52695017420, -46383583820, -43227910220, -40072236620, -33760803020, -30605129420, -27449455820, -21138022220, -17982348620, -14826675020, -12220073420, -12219987020, -12219900620, -12219814220, -12219727820, -12219641420, -12219555020, -12219468620, -12219382220, -12219295820, -2398294220 ], - "diffs" : [ -173380, -86980, -580, 85820, 172220, 258620, 345020, 431420, 517820, 604220, 690620, 777020, 863420, 777020, 690620, 604220, 517820, 431420, 345020, 258620, 172220, 85820, -580, 0 ] + "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ], + "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ] }, { "tz" : "Europe/Dublin", - "switches" : [ -62135595300, -59006360100, -55850686500, -52695012900, -46383579300, -43227905700, -40072232100, -33760798500, -30605124900, -27449451300, -21138017700, -17982344100, -14826670500, -12220068900, -12219982500, -12219896100, -12219809700, -12219723300, -12219636900, -12219550500, -12219464100, -12219377700, -12219291300, -2821649700, -2208988800 ], - "diffs" : [ -174300, -87900, -1500, 84900, 171300, 257700, 344100, 430500, 516900, 603300, 689700, 776100, 862500, 776100, 689700, 603300, 516900, 430500, 344100, 257700, 171300, 84900, -1500, -1521, 0 ] + "switches" : [ -62135595279, -59006360079, -55850686479, -52695012879, -46383579279, -43227905679, -40072232079, -33760798479, -30605124879, -27449451279, -21138017679, -17982344079, -14826670479, -12220068879, -12219982479, -12219896079, -12219809679, -12219723279, -12219636879, -12219550479, -12219464079, -12219377679, -12219291279, -2208988800 ], + "diffs" : [ -174321, -87921, -1521, 84879, 171279, 257679, 344079, 430479, 516879, 603279, 689679, 776079, 862479, 776079, 689679, 603279, 516879, 430479, 344079, 257679, 171279, 84879, -1521, 0 ] }, { "tz" : "Europe/Gibraltar", "switches" : [ -62135595516, -59006360316, -55850686716, -52695013116, -46383579516, -43227905916, -40072232316, -33760798716, -30605125116, -27449451516, -21138017916, -17982344316, -14826670716, -12220069116, -12219982716, -12219896316, -12219809916, -12219723516, -12219637116, -12219550716, -12219464316, -12219377916, -12219291516, -2821649916, -2208988800 ], @@ -1786,6 +1790,10 @@ "tz" : "Europe/Kirov", "switches" : [ -62135608728, -59006373528, -55850699928, -52695026328, -46383592728, -43227919128, -40072245528, -33760811928, -30605138328, -27449464728, -21138031128, -17982357528, -14826683928, -12220082328, -12219995928, -12219909528, -12219823128, -12219736728, -12219650328, -12219563928, -12219477528, -12219391128, -12219304728, -2208988800 ], "diffs" : [ -171672, -85272, 1128, 87528, 173928, 260328, 346728, 433128, 519528, 605928, 692328, 778728, 865128, 778728, 692328, 605928, 519528, 433128, 346728, 260328, 173928, 87528, 1128, 0 ] +}, { + "tz" : "Europe/Kyiv", + "switches" : [ -62135604124, -59006368924, -55850695324, -52695021724, -46383588124, -43227914524, -40072240924, -33760807324, -30605133724, -27449460124, -21138026524, -17982352924, -14826679324, -12220077724, -12219991324, -12219904924, -12219818524, -12219732124, -12219645724, -12219559324, -12219472924, -12219386524, -12219300124, -2208988800 ], + "diffs" : [ -172676, -86276, 124, 86524, 172924, 259324, 345724, 432124, 518524, 604924, 691324, 777724, 864124, 777724, 691324, 604924, 518524, 432124, 345724, 259324, 172924, 86524, 124, 0 ] }, { "tz" : "Europe/Lisbon", "switches" : [ -62135594595, -59006359395, -55850685795, -52695012195, -46383578595, -43227904995, -40072231395, -33760797795, -30605124195, -27449450595, -21138016995, -17982343395, -14826669795, -12220068195, -12219981795, -12219895395, -12219808995, -12219722595, -12219636195, -12219549795, -12219463395, -12219376995, -12219290595, -2208988800 ], @@ -1800,8 +1808,8 @@ "diffs" : [ -172875, -86475, -75, 86325, 172725, 259125, 345525, 431925, 518325, 604725, 691125, 777525, 863925, 777525, 691125, 604725, 518325, 431925, 345525, 259125, 172725, 86325, -75, 0 ] }, { "tz" : "Europe/Luxembourg", - "switches" : [ -62135598276, -59006363076, -55850689476, -52695015876, -46383582276, -43227908676, -40072235076, -33760801476, -30605127876, -27449454276, -21138020676, -17982347076, -14826673476, -12220071876, -12219985476, -12219899076, -12219812676, -12219726276, -12219639876, -12219553476, -12219467076, -12219380676, -12219294276, -2208988800 ], - "diffs" : [ -174924, -88524, -2124, 84276, 170676, 257076, 343476, 429876, 516276, 602676, 689076, 775476, 861876, 775476, 689076, 602676, 516276, 429876, 343476, 257076, 170676, 84276, -2124, 0 ] + "switches" : [ -62135597850, -59006362650, -55850689050, -52695015450, -46383581850, -43227908250, -40072234650, -33760801050, -30605127450, -27449453850, -21138020250, -17982346650, -14826673050, -12220071450, -12219985050, -12219898650, -12219812250, -12219725850, -12219639450, -12219553050, -12219466650, -12219380250, -12219293850, -2450995200, -2208988800 ], + "diffs" : [ -175350, -88950, -2550, 83850, 170250, 256650, 343050, 429450, 515850, 602250, 688650, 775050, 861450, 775050, 688650, 602250, 515850, 429450, 343050, 256650, 170250, 83850, -2550, -3600, 0 ] }, { "tz" : "Europe/Madrid", "switches" : [ -62135595916, -59006360716, -55850687116, -52695013516, -46383579916, -43227906316, -40072232716, -33760799116, -30605125516, -27449451916, -21138018316, -17982344716, -14826671116, -12220069516, -12219983116, -12219896716, -12219810316, -12219723916, -12219637516, -12219551116, -12219464716, -12219378316, -12219291916, -2208988800 ], @@ -1820,8 +1828,8 @@ "diffs" : [ -176984, -90584, -4184, 82216, 168616, 255016, 341416, 427816, 514216, 600616, 687016, 773416, 859816, 773416, 687016, 600616, 514216, 427816, 341416, 255016, 168616, 82216, -4184, -4200, 0 ] }, { "tz" : "Europe/Monaco", - "switches" : [ -62135598572, -59006363372, -55850689772, -52695016172, -46383582572, -43227908972, -40072235372, -33760801772, -30605128172, -27449454572, -21138020972, -17982347372, -14826673772, -12220072172, -12219985772, -12219899372, -12219812972, -12219726572, -12219640172, -12219553772, -12219467372, -12219380972, -12219294572, -2486680172, -2208988800 ], - "diffs" : [ -174628, -88228, -1828, 84572, 170972, 257372, 343772, 430172, 516572, 602972, 689372, 775772, 862172, 775772, 689372, 602972, 516572, 430172, 343772, 257372, 170972, 84572, -1828, -3039, 0 ] + "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12220070961, -12219984561, -12219898161, -12219811761, -12219725361, -12219638961, -12219552561, -12219466161, -12219379761, -12219293361, -2208988800 ], + "diffs" : [ -175839, -89439, -3039, 83361, 169761, 256161, 342561, 428961, 515361, 601761, 688161, 774561, 860961, 774561, 688161, 601761, 515361, 428961, 342561, 256161, 169761, 83361, -3039, 0 ] }, { "tz" : "Europe/Moscow", "switches" : [ -62135605817, -59006370617, -55850697017, -52695023417, -46383589817, -43227916217, -40072242617, -33760809017, -30605135417, -27449461817, -21138028217, -17982354617, -14826681017, -12220079417, -12219993017, -12219906617, -12219820217, -12219733817, -12219647417, -12219561017, -12219474617, -12219388217, -12219301817, -2208988800 ], @@ -1832,8 +1840,8 @@ "diffs" : [ -171992, -85592, 808, 87208, 173608, 260008, 346408, 432808, 519208, 605608, 692008, 778408, 864808, 778408, 692008, 605608, 519208, 432808, 346408, 260008, 173608, 87208, 808, 0 ] }, { "tz" : "Europe/Oslo", - "switches" : [ -62135599380, -59006364180, -55850690580, -52695016980, -46383583380, -43227909780, -40072236180, -33760802580, -30605128980, -27449455380, -21138021780, -17982348180, -14826674580, -12220072980, -12219986580, -12219900180, -12219813780, -12219727380, -12219640980, -12219554580, -12219468180, -12219381780, -12219295380, -2366757780 ], - "diffs" : [ -173820, -87420, -1020, 85380, 171780, 258180, 344580, 430980, 517380, 603780, 690180, 776580, 862980, 776580, 690180, 603780, 517380, 430980, 344580, 258180, 171780, 85380, -1020, 0 ] + "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ], + "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ] }, { "tz" : "Europe/Paris", "switches" : [ -62135597361, -59006362161, -55850688561, -52695014961, -46383581361, -43227907761, -40072234161, -33760800561, -30605126961, -27449453361, -21138019761, -17982346161, -14826672561, -12220070961, -12219984561, -12219898161, -12219811761, -12219725361, -12219638961, -12219552561, -12219466161, -12219379761, -12219293361, -2208988800 ], @@ -1884,8 +1892,8 @@ "diffs" : [ -174404, -88004, -1604, 84796, 171196, 257596, 343996, 430396, 516796, 603196, 689596, 775996, 862396, 775996, 689596, 603196, 516796, 430396, 343996, 257596, 171196, 84796, -1604, -184, 0 ] }, { "tz" : "Europe/Stockholm", - "switches" : [ -62135601132, -59006365932, -55850692332, -52695018732, -46383585132, -43227911532, -40072237932, -33760804332, -30605130732, -27449457132, -21138023532, -17982349932, -14826676332, -12220074732, -12219988332, -12219901932, -12219815532, -12219729132, -12219642732, -12219556332, -12219469932, -12219383532, -12219297132, -2871681132, -2208992414 ], - "diffs" : [ -172068, -85668, 732, 87132, 173532, 259932, 346332, 432732, 519132, 605532, 691932, 778332, 864732, 778332, 691932, 605532, 519132, 432732, 346332, 259932, 173532, 87132, 732, 14, 0 ] + "switches" : [ -62135600008, -59006364808, -55850691208, -52695017608, -46383584008, -43227910408, -40072236808, -33760803208, -30605129608, -27449456008, -21138022408, -17982348808, -14826675208, -12220073608, -12219987208, -12219900808, -12219814408, -12219728008, -12219641608, -12219555208, -12219468808, -12219382408, -12219296008, -2422054408 ], + "diffs" : [ -173192, -86792, -392, 86008, 172408, 258808, 345208, 431608, 518008, 604408, 690808, 777208, 863608, 777208, 690808, 604408, 518008, 431608, 345208, 258808, 172408, 86008, -392, 0 ] }, { "tz" : "Europe/Tallinn", "switches" : [ -62135602740, -59006367540, -55850693940, -52695020340, -46383586740, -43227913140, -40072239540, -33760805940, -30605132340, -27449458740, -21138025140, -17982351540, -14826677940, -12220076340, -12219989940, -12219903540, -12219817140, -12219730740, -12219644340, -12219557940, -12219471540, -12219385140, -12219298740, -2208988800 ], @@ -1925,7 +1933,7 @@ }, { "tz" : "Europe/Volgograd", "switches" : [ -62135607460, -59006372260, -55850698660, -52695025060, -46383591460, -43227917860, -40072244260, -33760810660, -30605137060, -27449463460, -21138029860, -17982356260, -14826682660, -12220081060, -12219994660, -12219908260, -12219821860, -12219735460, -12219649060, -12219562660, -12219476260, -12219389860, -12219303460, -2208988800 ], - "diffs" : [ -176540, -90140, -3740, 82660, 169060, 255460, 341860, 428260, 514660, 601060, 687460, 773860, 860260, 773860, 687460, 601060, 514660, 428260, 341860, 255460, 169060, 82660, -3740, 0 ] + "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, 0 ] }, { "tz" : "Europe/Warsaw", "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ], @@ -1968,8 +1976,8 @@ "diffs" : [ -174198, -87798, -1398, 85002, 171402, 257802, 344202, 430602, 517002, 603402, 689802, 776202, 862602, 776202, 689802, 603402, 517002, 430602, 344202, 257802, 171402, 85002, -1398, 0 ] }, { "tz" : "Iceland", - "switches" : [ -62135591520, -59006356320, -55850682720, -52695009120, -46383575520, -43227901920, -40072228320, -33760794720, -30605121120, -27449447520, -21138013920, -17982340320, -14826666720, -12220065120, -12219978720, -12219892320, -12219805920, -12219719520, -12219633120, -12219546720, -12219460320, -12219373920, -12219287520, -2208988800 ], - "diffs" : [ -178080, -91680, -5280, 81120, 167520, 253920, 340320, 426720, 513120, 599520, 685920, 772320, 858720, 772320, 685920, 599520, 513120, 426720, 340320, 253920, 167520, 81120, -5280, 0 ] + "switches" : [ -62135595832, -59006360632, -55850687032, -52695013432, -46383579832, -43227906232, -40072232632, -33760799032, -30605125432, -27449451832, -21138018232, -17982344632, -14826671032, -12220069432, -12219983032, -12219896632, -12219810232, -12219723832, -12219637432, -12219551032, -12219464632, -12219378232, -12219291832, -2208988800 ], + "diffs" : [ -173768, -87368, -968, 85432, 171832, 258232, 344632, 431032, 517432, 603832, 690232, 776632, 863032, 776632, 690232, 603832, 517432, 431032, 344632, 258232, 171832, 85432, -968, 0 ] }, { "tz" : "Indian/Antananarivo", "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ], @@ -1980,24 +1988,24 @@ "diffs" : [ -177020, -90620, -4220, 82180, 168580, 254980, 341380, 427780, 514180, 600580, 686980, 773380, 859780, 773380, 686980, 600580, 514180, 427780, 341380, 254980, 168580, 82180, -4220, 0 ] }, { "tz" : "Indian/Christmas", - "switches" : [ -62135622172, -59006386972, -55850713372, -52695039772, -46383606172, -43227932572, -40072258972, -33760825372, -30605151772, -27449478172, -21138044572, -17982370972, -14826697372, -12220095772, -12220009372, -12219922972, -12219836572, -12219750172, -12219663772, -12219577372, -12219490972, -12219404572, -12219318172, -2364102172 ], - "diffs" : [ -172628, -86228, 172, 86572, 172972, 259372, 345772, 432172, 518572, 604972, 691372, 777772, 864172, 777772, 691372, 604972, 518572, 432172, 345772, 259372, 172972, 86572, 172, 0 ] + "switches" : [ -62135620924, -59006385724, -55850712124, -52695038524, -46383604924, -43227931324, -40072257724, -33760824124, -30605150524, -27449476924, -21138043324, -17982369724, -14826696124, -12220094524, -12220008124, -12219921724, -12219835324, -12219748924, -12219662524, -12219576124, -12219489724, -12219403324, -12219316924, -2208988800 ], + "diffs" : [ -173876, -87476, -1076, 85324, 171724, 258124, 344524, 430924, 517324, 603724, 690124, 776524, 862924, 776524, 690124, 603724, 517324, 430924, 344524, 258124, 171724, 85324, -1076, 0 ] }, { "tz" : "Indian/Cocos", - "switches" : [ -62135620060, -59006384860, -55850711260, -52695037660, -46383604060, -43227930460, -40072256860, -33760823260, -30605149660, -27449476060, -21138042460, -17982368860, -14826695260, -12220093660, -12220007260, -12219920860, -12219834460, -12219748060, -12219661660, -12219575260, -12219488860, -12219402460, -12219316060, -2209012060 ], - "diffs" : [ -172940, -86540, -140, 86260, 172660, 259060, 345460, 431860, 518260, 604660, 691060, 777460, 863860, 777460, 691060, 604660, 518260, 431860, 345460, 259060, 172660, 86260, -140, 0 ] + "switches" : [ -62135619887, -59006384687, -55850711087, -52695037487, -46383603887, -43227930287, -40072256687, -33760823087, -30605149487, -27449475887, -21138042287, -17982368687, -14826695087, -12220093487, -12220007087, -12219920687, -12219834287, -12219747887, -12219661487, -12219575087, -12219488687, -12219402287, -12219315887, -2208988800 ], + "diffs" : [ -173113, -86713, -313, 86087, 172487, 258887, 345287, 431687, 518087, 604487, 690887, 777287, 863687, 777287, 690887, 604487, 518087, 431687, 345287, 258887, 172487, 86087, -313, 0 ] }, { "tz" : "Indian/Comoro", "switches" : [ -62135605636, -59006370436, -55850696836, -52695023236, -46383589636, -43227916036, -40072242436, -33760808836, -30605135236, -27449461636, -21138028036, -17982354436, -14826680836, -12220079236, -12219992836, -12219906436, -12219820036, -12219733636, -12219647236, -12219560836, -12219474436, -12219388036, -12219301636, -2208988800 ], "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ] }, { "tz" : "Indian/Kerguelen", - "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], - "diffs" : [ -190800, -104400, -18000, 68400, 154800, 241200, 327600, 414000, 500400, 586800, 673200, 759600, 846000, 759600, 673200, 586800, 500400, 414000, 327600, 241200, 154800, 68400, -18000, 0 ] + "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12220088040, -12220001640, -12219915240, -12219828840, -12219742440, -12219656040, -12219569640, -12219483240, -12219396840, -12219310440, -2208988800 ], + "diffs" : [ -173160, -86760, -360, 86040, 172440, 258840, 345240, 431640, 518040, 604440, 690840, 777240, 863640, 777240, 690840, 604440, 518040, 431640, 345240, 258840, 172440, 86040, -360, 0 ] }, { "tz" : "Indian/Mahe", - "switches" : [ -62135610108, -59006374908, -55850701308, -52695027708, -46383594108, -43227920508, -40072246908, -33760813308, -30605139708, -27449466108, -21138032508, -17982358908, -14826685308, -12220083708, -12219997308, -12219910908, -12219824508, -12219738108, -12219651708, -12219565308, -12219478908, -12219392508, -12219306108, -2208988800 ], - "diffs" : [ -173892, -87492, -1092, 85308, 171708, 258108, 344508, 430908, 517308, 603708, 690108, 776508, 862908, 776508, 690108, 603708, 517308, 430908, 344508, 258108, 171708, 85308, -1092, 0 ] + "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ], + "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ] }, { "tz" : "Indian/Maldives", "switches" : [ -62135614440, -59006379240, -55850705640, -52695032040, -46383598440, -43227924840, -40072251240, -33760817640, -30605144040, -27449470440, -21138036840, -17982363240, -14826689640, -12220088040, -12220001640, -12219915240, -12219828840, -12219742440, -12219656040, -12219569640, -12219483240, -12219396840, -12219310440, -2208988800 ], @@ -2012,8 +2020,8 @@ "diffs" : [ -174764, -88364, -1964, 84436, 170836, 257236, 343636, 430036, 516436, 602836, 689236, 775636, 862036, 775636, 689236, 602836, 516436, 430036, 343636, 257236, 170836, 84436, -1964, 0 ] }, { "tz" : "Indian/Reunion", - "switches" : [ -62135610112, -59006374912, -55850701312, -52695027712, -46383594112, -43227920512, -40072246912, -33760813312, -30605139712, -27449466112, -21138032512, -17982358912, -14826685312, -12220083712, -12219997312, -12219910912, -12219824512, -12219738112, -12219651712, -12219565312, -12219478912, -12219392512, -12219306112, -2208988800 ], - "diffs" : [ -173888, -87488, -1088, 85312, 171712, 258112, 344512, 430912, 517312, 603712, 690112, 776512, 862912, 776512, 690112, 603712, 517312, 430912, 344512, 258112, 171712, 85312, -1088, 0 ] + "switches" : [ -62135610072, -59006374872, -55850701272, -52695027672, -46383594072, -43227920472, -40072246872, -33760813272, -30605139672, -27449466072, -21138032472, -17982358872, -14826685272, -12220083672, -12219997272, -12219910872, -12219824472, -12219738072, -12219651672, -12219565272, -12219478872, -12219392472, -12219306072, -2208988800 ], + "diffs" : [ -173928, -87528, -1128, 85272, 171672, 258072, 344472, 430872, 517272, 603672, 690072, 776472, 862872, 776472, 690072, 603672, 517272, 430872, 344472, 258072, 171672, 85272, -1128, 0 ] }, { "tz" : "Iran", "switches" : [ -62135609144, -59006373944, -55850700344, -52695026744, -46383593144, -43227919544, -40072245944, -33760812344, -30605138744, -27449465144, -21138031544, -17982357944, -14826684344, -12220082744, -12219996344, -12219909944, -12219823544, -12219737144, -12219650744, -12219564344, -12219477944, -12219391544, -12219305144, -2208988800 ], @@ -2096,8 +2104,8 @@ "diffs" : [ -174672, -88272, -1872, 84528, 170928, 257328, 343728, 430128, 516528, 602928, 689328, 775728, 862128, 775728, 689328, 602928, 516528, 430128, 343728, 257328, 170928, 84528, -1872, -1800, 0 ] }, { "tz" : "Pacific/Chuuk", - "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ], - "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ] + "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ], + "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ] }, { "tz" : "Pacific/Easter", "switches" : [ -62135570552, -59006335352, -55850661752, -52694988152, -46383554552, -43227880952, -40072207352, -33760773752, -30605100152, -27449426552, -21137992952, -17982319352, -14826645752, -12220044152, -12219957752, -12219871352, -12219784952, -12219698552, -12219612152, -12219525752, -12219439352, -12219352952, -12219266552, -2208988800 ], @@ -2108,8 +2116,8 @@ "diffs" : [ -172004, -85604, 796, 87196, 173596, 259996, 346396, 432796, 519196, 605596, 691996, 778396, 864796, 778396, 691996, 605596, 519196, 432796, 346396, 259996, 173596, 87196, 796, 0 ] }, { "tz" : "Pacific/Enderbury", - "switches" : [ -62135555740, -59006320540, -55850646940, -52694973340, -46383539740, -43227866140, -40072192540, -33760758940, -30605085340, -27449411740, -21137978140, -17982304540, -14826630940, -12220029340, -12219942940, -12219856540, -12219770140, -12219683740, -12219597340, -12219510940, -12219424540, -12219338140, -12219251740, -2208988800 ], - "diffs" : [ -260660, -174260, -87860, -1460, 84940, 171340, 257740, 344140, 430540, 516940, 603340, 689740, 776140, 689740, 603340, 516940, 430540, 344140, 257740, 171340, 84940, -1460, -87860, 0 ] + "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], + "diffs" : [ -219600, -133200, -46800, 39600, 126000, 212400, 298800, 385200, 471600, 558000, 644400, 730800, 817200, 730800, 644400, 558000, 471600, 385200, 298800, 212400, 126000, 39600, -46800, 0 ] }, { "tz" : "Pacific/Fakaofo", "switches" : [ -62135555704, -59006320504, -55850646904, -52694973304, -46383539704, -43227866104, -40072192504, -33760758904, -30605085304, -27449411704, -21137978104, -17982304504, -14826630904, -12220029304, -12219942904, -12219856504, -12219770104, -12219683704, -12219597304, -12219510904, -12219424504, -12219338104, -12219251704, -2208988800 ], @@ -2120,8 +2128,8 @@ "diffs" : [ -173056, -86656, -256, 86144, 172544, 258944, 345344, 431744, 518144, 604544, 690944, 777344, 863744, 777344, 690944, 604544, 518144, 431744, 345344, 258944, 172544, 86144, -256, 0 ] }, { "tz" : "Pacific/Funafuti", - "switches" : [ -62135639812, -59006404612, -55850731012, -52695057412, -46383623812, -43227950212, -40072276612, -33760843012, -30605169412, -27449495812, -21138062212, -17982388612, -14826715012, -12220113412, -12220027012, -12219940612, -12219854212, -12219767812, -12219681412, -12219595012, -12219508612, -12219422212, -12219335812, -2208988800 ], - "diffs" : [ -172988, -86588, -188, 86212, 172612, 259012, 345412, 431812, 518212, 604612, 691012, 777412, 863812, 777412, 691012, 604612, 518212, 431812, 345412, 259012, 172612, 86212, -188, 0 ] + "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ], + "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ] }, { "tz" : "Pacific/Galapagos", "switches" : [ -62135575296, -59006340096, -55850666496, -52694992896, -46383559296, -43227885696, -40072212096, -33760778496, -30605104896, -27449431296, -21137997696, -17982324096, -14826650496, -12220048896, -12219962496, -12219876096, -12219789696, -12219703296, -12219616896, -12219530496, -12219444096, -12219357696, -12219271296, -2208988800 ], @@ -2146,6 +2154,10 @@ "tz" : "Pacific/Johnston", "switches" : [ -62135558914, -59006323714, -55850650114, -52694976514, -46383542914, -43227869314, -40072195714, -33760762114, -30605088514, -27449414914, -21137981314, -17982307714, -14826634114, -12220032514, -12219946114, -12219859714, -12219773314, -12219686914, -12219600514, -12219514114, -12219427714, -12219341314, -12219254914, -2334101314, -2208988800 ], "diffs" : [ -174686, -88286, -1886, 84514, 170914, 257314, 343714, 430114, 516514, 602914, 689314, 775714, 862114, 775714, 689314, 602914, 516514, 430114, 343714, 257314, 170914, 84514, -1886, -1800, 0 ] +}, { + "tz" : "Pacific/Kanton", + "switches" : [ -62135596800, -59006361600, -55850688000, -52695014400, -46383580800, -43227907200, -40072233600, -33760800000, -30605126400, -27449452800, -21138019200, -17982345600, -14826672000, -12220070400, -12219984000, -12219897600, -12219811200, -12219724800, -12219638400, -12219552000, -12219465600, -12219379200, -12219292800, -2208988800 ], + "diffs" : [ -219600, -133200, -46800, 39600, 126000, 212400, 298800, 385200, 471600, 558000, 644400, 730800, 817200, 730800, 644400, 558000, 471600, 385200, 298800, 212400, 126000, 39600, -46800, 0 ] }, { "tz" : "Pacific/Kiritimati", "switches" : [ -62135559040, -59006323840, -55850650240, -52694976640, -46383543040, -43227869440, -40072195840, -33760762240, -30605088640, -27449415040, -21137981440, -17982307840, -14826634240, -12220032640, -12219946240, -12219859840, -12219773440, -12219687040, -12219600640, -12219514240, -12219427840, -12219341440, -12219255040, -2208988800 ], @@ -2160,8 +2172,8 @@ "diffs" : [ -175840, -89440, -3040, 83360, 169760, 256160, 342560, 428960, 515360, 601760, 688160, 774560, 860960, 774560, 688160, 601760, 515360, 428960, 342560, 256160, 169760, 83360, -3040, 0 ] }, { "tz" : "Pacific/Majuro", - "switches" : [ -62135637888, -59006402688, -55850729088, -52695055488, -46383621888, -43227948288, -40072274688, -33760841088, -30605167488, -27449493888, -21138060288, -17982386688, -14826713088, -12220111488, -12220025088, -12219938688, -12219852288, -12219765888, -12219679488, -12219593088, -12219506688, -12219420288, -12219333888, -2208988800 ], - "diffs" : [ -174912, -88512, -2112, 84288, 170688, 257088, 343488, 429888, 516288, 602688, 689088, 775488, 861888, 775488, 689088, 602688, 516288, 429888, 343488, 257088, 170688, 84288, -2112, 0 ] + "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ], + "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ] }, { "tz" : "Pacific/Marquesas", "switches" : [ -62135563320, -59006328120, -55850654520, -52694980920, -46383547320, -43227873720, -40072200120, -33760766520, -30605092920, -27449419320, -21137985720, -17982312120, -14826638520, -12220036920, -12219950520, -12219864120, -12219777720, -12219691320, -12219604920, -12219518520, -12219432120, -12219345720, -12219259320, -2208988800 ], @@ -2200,20 +2212,20 @@ "diffs" : [ -175220, -88820, -2420, 83980, 170380, 256780, 343180, 429580, 515980, 602380, 688780, 775180, 861580, 775180, 688780, 602380, 515980, 429580, 343180, 256780, 170380, 83980, -2420, 0 ] }, { "tz" : "Pacific/Pohnpei", - "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ], - "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ] + "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12220108788, -12220022388, -12219935988, -12219849588, -12219763188, -12219676788, -12219590388, -12219503988, -12219417588, -12219331188, -2208988800 ], + "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, 776388, 689988, 603588, 517188, 430788, 344388, 257988, 171588, 85188, -1212, 0 ] }, { "tz" : "Pacific/Ponape", - "switches" : [ -62135548372, -59006313172, -55850639572, -52694965972, -46383532372, -43227858772, -40072185172, -33760751572, -30605077972, -27449404372, -21137970772, -17982297172, -14826623572, -12220021972, -12219935572, -12219849172, -12219762772, -12219676372, -12219589972, -12219503572, -12219417172, -12219330772, -12219244372, -3944629972, -2208988800 ], - "diffs" : [ -260828, -174428, -88028, -1628, 84772, 171172, 257572, 343972, 430372, 516772, 603172, 689572, 775972, 689572, 603172, 516772, 430372, 343972, 257572, 171172, 84772, -1628, -88028, -1628, 0 ] + "switches" : [ -62135635188, -59006399988, -55850726388, -52695052788, -46383619188, -43227945588, -40072271988, -33760838388, -30605164788, -27449491188, -21138057588, -17982383988, -14826710388, -12220108788, -12220022388, -12219935988, -12219849588, -12219763188, -12219676788, -12219590388, -12219503988, -12219417588, -12219331188, -2208988800 ], + "diffs" : [ -174012, -87612, -1212, 85188, 171588, 257988, 344388, 430788, 517188, 603588, 689988, 776388, 862788, 776388, 689988, 603588, 517188, 430788, 344388, 257988, 171588, 85188, -1212, 0 ] }, { "tz" : "Pacific/Port_Moresby", "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ], "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ] }, { "tz" : "Pacific/Rarotonga", - "switches" : [ -62135558456, -59006323256, -55850649656, -52694976056, -46383542456, -43227868856, -40072195256, -33760761656, -30605088056, -27449414456, -21137980856, -17982307256, -14826633656, -12220032056, -12219945656, -12219859256, -12219772856, -12219686456, -12219600056, -12219513656, -12219427256, -12219340856, -12219254456, -2208988800 ], - "diffs" : [ -175144, -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, 775256, 688856, 602456, 516056, 429656, 343256, 256856, 170456, 84056, -2344, 0 ] + "switches" : [ -62135644856, -59006409656, -55850736056, -52695062456, -46383628856, -43227955256, -40072281656, -33760848056, -30605174456, -27449500856, -21138067256, -17982393656, -14826720056, -12220118456, -12220032056, -12219945656, -12219859256, -12219772856, -12219686456, -12219600056, -12219513656, -12219427256, -12219340856, -2209555256, -2208988800 ], + "diffs" : [ -88744, -2344, 84056, 170456, 256856, 343256, 429656, 516056, 602456, 688856, 775256, 861656, 948056, 861656, 775256, 688856, 602456, 516056, 429656, 343256, 256856, 170456, 84056, -2344, 0 ] }, { "tz" : "Pacific/Saipan", "switches" : [ -62135545140, -59006309940, -55850636340, -52694962740, -46383529140, -43227855540, -40072181940, -33760748340, -30605074740, -27449401140, -21137967540, -17982293940, -14826620340, -12220018740, -12219932340, -12219845940, -12219759540, -12219673140, -12219586740, -12219500340, -12219413940, -12219327540, -12219241140, -3944626740, -2208988800 ], @@ -2232,24 +2244,24 @@ "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ] }, { "tz" : "Pacific/Tongatapu", - "switches" : [ -62135641160, -59006405960, -55850732360, -52695058760, -46383625160, -43227951560, -40072277960, -33760844360, -30605170760, -27449497160, -21138063560, -17982389960, -14826716360, -12220114760, -12220028360, -12219941960, -12219855560, -12219769160, -12219682760, -12219596360, -12219509960, -12219423560, -12219337160, -2208988800 ], - "diffs" : [ -175240, -88840, -2440, 83960, 170360, 256760, 343160, 429560, 515960, 602360, 688760, 775160, 861560, 775160, 688760, 602360, 515960, 429560, 343160, 256760, 170360, 83960, -2440, 0 ] + "switches" : [ -62135641152, -59006405952, -55850732352, -52695058752, -46383625152, -43227951552, -40072277952, -33760844352, -30605170752, -27449497152, -21138063552, -17982389952, -14826716352, -12220114752, -12220028352, -12219941952, -12219855552, -12219769152, -12219682752, -12219596352, -12219509952, -12219423552, -12219337152, -2208988800 ], + "diffs" : [ -175248, -88848, -2448, 83952, 170352, 256752, 343152, 429552, 515952, 602352, 688752, 775152, 861552, 775152, 688752, 602352, 515952, 429552, 343152, 256752, 170352, 83952, -2448, 0 ] }, { "tz" : "Pacific/Truk", - "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ], - "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ] + "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ], + "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ] }, { "tz" : "Pacific/Wake", - "switches" : [ -62135636788, -59006401588, -55850727988, -52695054388, -46383620788, -43227947188, -40072273588, -33760839988, -30605166388, -27449492788, -21138059188, -17982385588, -14826711988, -12220110388, -12220023988, -12219937588, -12219851188, -12219764788, -12219678388, -12219591988, -12219505588, -12219419188, -12219332788, -2208988800 ], - "diffs" : [ -176012, -89612, -3212, 83188, 169588, 255988, 342388, 428788, 515188, 601588, 687988, 774388, 860788, 774388, 687988, 601588, 515188, 428788, 342388, 255988, 169588, 83188, -3212, 0 ] + "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ], + "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ] }, { "tz" : "Pacific/Wallis", - "switches" : [ -62135640920, -59006405720, -55850732120, -52695058520, -46383624920, -43227951320, -40072277720, -33760844120, -30605170520, -27449496920, -21138063320, -17982389720, -14826716120, -12220114520, -12220028120, -12219941720, -12219855320, -12219768920, -12219682520, -12219596120, -12219509720, -12219423320, -12219336920, -2208988800 ], - "diffs" : [ -171880, -85480, 920, 87320, 173720, 260120, 346520, 432920, 519320, 605720, 692120, 778520, 864920, 778520, 692120, 605720, 519320, 432920, 346520, 260120, 173720, 87320, 920, 0 ] + "switches" : [ -62135638324, -59006403124, -55850729524, -52695055924, -46383622324, -43227948724, -40072275124, -33760841524, -30605167924, -27449494324, -21138060724, -17982387124, -14826713524, -12220111924, -12220025524, -12219939124, -12219852724, -12219766324, -12219679924, -12219593524, -12219507124, -12219420724, -12219334324, -2208988800 ], + "diffs" : [ -174476, -88076, -1676, 84724, 171124, 257524, 343924, 430324, 516724, 603124, 689524, 775924, 862324, 775924, 689524, 603124, 516724, 430324, 343924, 257524, 171124, 84724, -1676, 0 ] }, { "tz" : "Pacific/Yap", - "switches" : [ -62135546828, -59006311628, -55850638028, -52694964428, -46383530828, -43227857228, -40072183628, -33760750028, -30605076428, -27449402828, -21137969228, -17982295628, -14826622028, -12220020428, -12219934028, -12219847628, -12219761228, -12219674828, -12219588428, -12219502028, -12219415628, -12219329228, -12219242828, -3944628428, -2208988800 ], - "diffs" : [ -258772, -172372, -85972, 428, 86828, 173228, 259628, 346028, 432428, 518828, 605228, 691628, 778028, 691628, 605228, 518828, 432428, 346028, 259628, 173228, 86828, 428, -85972, 428, 0 ] + "switches" : [ -62135632120, -59006396920, -55850723320, -52695049720, -46383616120, -43227942520, -40072268920, -33760835320, -30605161720, -27449488120, -21138054520, -17982380920, -14826707320, -12220105720, -12220019320, -12219932920, -12219846520, -12219760120, -12219673720, -12219587320, -12219500920, -12219414520, -12219328120, -2840176120, -2366790512 ], + "diffs" : [ -173480, -87080, -680, 85720, 172120, 258520, 344920, 431320, 517720, 604120, 690520, 776920, 863320, 776920, 690520, 604120, 517720, 431320, 344920, 258520, 172120, 85720, -680, -688, 0 ] }, { "tz" : "Poland", "switches" : [ -62135601840, -59006366640, -55850693040, -52695019440, -46383585840, -43227912240, -40072238640, -33760805040, -30605131440, -27449457840, -21138024240, -17982350640, -14826677040, -12220075440, -12219989040, -12219902640, -12219816240, -12219729840, -12219643440, -12219557040, -12219470640, -12219384240, -12219297840, -2208988800 ], @@ -2370,10 +2382,6 @@ "tz" : "US/Pacific", "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ], "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ] -}, { - "tz" : "US/Pacific-New", - "switches" : [ -62135568422, -59006333222, -55850659622, -52694986022, -46383552422, -43227878822, -40072205222, -33760771622, -30605098022, -27449424422, -21137990822, -17982317222, -14826643622, -12220042022, -12219955622, -12219869222, -12219782822, -12219696422, -12219610022, -12219523622, -12219437222, -12219350822, -12219264422, -2717640000 ], - "diffs" : [ -172378, -85978, 422, 86822, 173222, 259622, 346022, 432422, 518822, 605222, 691622, 778022, 864422, 778022, 691622, 605222, 518822, 432422, 346022, 259622, 173222, 86822, 422, 0 ] }, { "tz" : "US/Samoa", "switches" : [ -62135642232, -59006407032, -55850733432, -52695059832, -46383626232, -43227952632, -40072279032, -33760845432, -30605171832, -27449498232, -21138064632, -17982391032, -14826717432, -12220115832, -12220029432, -12219943032, -12219856632, -12219770232, -12219683832, -12219597432, -12219511032, -12219424632, -12219338232, -2445424632, -2208988800 ], diff --git a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json index eddc16f9ae76..6cf3132cc4ea 100644 --- a/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json +++ b/sql/catalyst/src/main/resources/julian-gregorian-rebase-micros.json @@ -5,14 +5,14 @@ }, { "tz" : "Africa/Accra", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], - "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ] + "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ] }, { "tz" : "Africa/Addis_Ababa", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], "diffs" : [ 174764, 88364, 1964, -84436, -170836, -257236, -343636, -430036, -516436, -602836, -689236, -775636, -862036, 1964, 0 ] }, { "tz" : "Africa/Algiers", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486682111, -2208988800 ], + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486595771, -2208988800 ], "diffs" : [ 175668, 89268, 2868, -83532, -169932, -256332, -342732, -429132, -515532, -601932, -688332, -774732, -861132, 2868, 3039, 0 ] }, { "tz" : "Africa/Asmara", @@ -29,7 +29,7 @@ }, { "tz" : "Africa/Bangui", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Banjul", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -45,7 +45,7 @@ }, { "tz" : "Africa/Brazzaville", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Bujumbura", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], @@ -81,7 +81,7 @@ }, { "tz" : "Africa/Douala", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/El_Aaiun", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -104,8 +104,8 @@ "diffs" : [ 173280, 86880, 480, -85920, -172320, -258720, -345120, -431520, -517920, -604320, -690720, -777120, -863520, 480, 1800, 0 ] }, { "tz" : "Africa/Juba", - "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], - "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ] + "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], + "diffs" : [ 172412, 86012, -388, -86788, -173188, -259588, -345988, -432388, -518788, -605188, -691588, -777988, -864388, -388, 0 ] }, { "tz" : "Africa/Kampala", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], @@ -121,15 +121,15 @@ }, { "tz" : "Africa/Kinshasa", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Lagos", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Libreville", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Lome", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -137,7 +137,7 @@ }, { "tz" : "Africa/Luanda", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Lubumbashi", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], @@ -149,7 +149,7 @@ }, { "tz" : "Africa/Malabo", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Maputo", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], @@ -181,7 +181,7 @@ }, { "tz" : "Africa/Niamey", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Nouakchott", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -193,7 +193,7 @@ }, { "tz" : "Africa/Porto-Novo", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175584, 89184, 2784, -83616, -170016, -256416, -342816, -429216, -515616, -602016, -688416, -774816, -861216, 2784, 0 ] + "diffs" : [ 175585, 89185, 2785, -83615, -170015, -256415, -342815, -429215, -515615, -602015, -688415, -774815, -861215, 2785, 0 ] }, { "tz" : "Africa/Sao_Tome", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2713914221, -2208988800 ], @@ -224,12 +224,12 @@ "diffs" : [ 89976, 3576, -82824, -169224, -255624, -342024, -428424, -514824, -601224, -687624, -774024, -860424, -946824, -82824, 3576, 0 ] }, { "tz" : "America/Anguilla", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Antigua", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Araguaina", "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ], @@ -288,16 +288,16 @@ "diffs" : [ 178392, 91992, 5592, -80808, -167208, -253608, -340008, -426408, -512808, -599208, -685608, -772008, -858408, 5592, 4608, 0 ] }, { "tz" : "America/Aruba", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Asuncion", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], "diffs" : [ 172240, 85840, -560, -86960, -173360, -259760, -346160, -432560, -518960, -605360, -691760, -778160, -864560, -560, 0 ] }, { "tz" : "America/Atikokan", - "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ], - "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ] + "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ], + "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ] }, { "tz" : "America/Atka", "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -3225230125, -2208988800 ], @@ -324,8 +324,8 @@ "diffs" : [ 172368, 85968, -432, -86832, -173232, -259632, -346032, -432432, -518832, -605232, -691632, -778032, -864432, -432, 0 ] }, { "tz" : "America/Blanc-Sablon", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2713896692 ], - "diffs" : [ 172108, 85708, -692, -87092, -173492, -259892, -346292, -432692, -519092, -605492, -691892, -778292, -864692, -692, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Boa_Vista", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], @@ -380,8 +380,8 @@ "diffs" : [ 173060, 86660, 260, -86140, -172540, -258940, -345340, -431740, -518140, -604540, -690940, -777340, -863740, 260, 0 ] }, { "tz" : "America/Coral_Harbour", - "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736812, -2208988800 ], - "diffs" : [ 176788, 90388, 3988, -82412, -168812, -255212, -341612, -428012, -514412, -600812, -687212, -773612, -860012, 3988, 3600, 0 ] + "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2524503688, -2208988800 ], + "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 1176, 0 ] }, { "tz" : "America/Cordoba", "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ], @@ -392,24 +392,24 @@ "diffs" : [ 171373, 84973, -1427, -87827, -174227, -260627, -347027, -433427, -519827, -606227, -692627, -779027, -865427, -1427, 0 ] }, { "tz" : "America/Creston", - "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713882436 ], - "diffs" : [ 175564, 89164, 2764, -83636, -170036, -256436, -342836, -429236, -515636, -602036, -688436, -774836, -861236, 2764, 0 ] + "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2717643600 ], + "diffs" : [ 174498, 88098, 1698, -84702, -171102, -257502, -343902, -430302, -516702, -603102, -689502, -775902, -862302, 1698, 0 ] }, { "tz" : "America/Cuiaba", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], "diffs" : [ 171860, 85460, -940, -87340, -173740, -260140, -346540, -432940, -519340, -605740, -692140, -778540, -864940, -940, 0 ] }, { "tz" : "America/Curacao", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Danmarkshavn", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], "diffs" : [ 177280, 90880, 4480, -81920, -168320, -254720, -341120, -427520, -513920, -600320, -686720, -773120, -859520, 4480, 0 ] }, { "tz" : "America/Dawson", - "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ], - "diffs" : [ 177460, 91060, 4660, -81740, -168140, -254540, -340940, -427340, -513740, -600140, -686540, -772940, -859340, 4660, 0 ] + "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ], + "diffs" : [ 181060, 94660, 8260, -78140, -164540, -250940, -337340, -423740, -510140, -596540, -682940, -769340, -855740, 8260, 0 ] }, { "tz" : "America/Dawson_Creek", "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2713885144, -2208988800 ], @@ -424,8 +424,8 @@ "diffs" : [ 174731, 88331, 1931, -84469, -170869, -257269, -343669, -430069, -516469, -602869, -689269, -775669, -862069, 1931, 0 ] }, { "tz" : "America/Dominica", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Edmonton", "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ], @@ -472,12 +472,12 @@ "diffs" : [ 171872, 85472, -928, -87328, -173728, -260128, -346528, -432928, -519328, -605728, -692128, -778528, -864928, -928, 430, 0 ] }, { "tz" : "America/Grenada", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Guadeloupe", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Guatemala", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ], @@ -489,7 +489,7 @@ }, { "tz" : "America/Guyana", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 172360, 85960, -440, -86840, -173240, -259640, -346040, -432440, -518840, -605240, -691640, -778040, -864440, -440, 0 ] + "diffs" : [ 172359, 85959, -441, -86841, -173241, -259641, -346041, -432441, -518841, -605241, -691641, -778041, -864441, -441, 0 ] }, { "tz" : "America/Halifax", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], @@ -572,8 +572,8 @@ "diffs" : [ 171990, 85590, -810, -87210, -173610, -260010, -346410, -432810, -519210, -605610, -692010, -778410, -864810, -810, 0 ] }, { "tz" : "America/Kralendijk", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/La_Paz", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], @@ -592,8 +592,8 @@ "diffs" : [ 175382, 88982, 2582, -83818, -170218, -256618, -343018, -429418, -515818, -602218, -688618, -775018, -861418, 2582, 3600, 0 ] }, { "tz" : "America/Lower_Princes", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 174947, 88547, 2147, -84253, -170653, -257053, -343453, -429853, -516253, -602653, -689053, -775453, -861853, 2147, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Maceio", "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ], @@ -608,8 +608,8 @@ "diffs" : [ 172804, 86404, 4, -86396, -172796, -259196, -345596, -431996, -518396, -604796, -691196, -777596, -863996, 4, 0 ] }, { "tz" : "America/Marigot", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Martinique", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], @@ -664,12 +664,12 @@ "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ] }, { "tz" : "America/Montserrat", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Nassau", - "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ], - "diffs" : [ 173370, 86970, 570, -85830, -172230, -258630, -345030, -431430, -517830, -604230, -690630, -777030, -863430, 570, 0 ] + "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2366736148 ], + "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ] }, { "tz" : "America/New_York", "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2717650800 ], @@ -698,6 +698,10 @@ "tz" : "America/North_Dakota/New_Salem", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2717647200, -2208988800 ], "diffs" : [ 175539, 89139, 2739, -83661, -170061, -256461, -342861, -429261, -515661, -602061, -688461, -774861, -861261, 2739, 3600, 0 ] +}, { + "tz" : "America/Nuuk", + "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2208988800 ], + "diffs" : [ 174416, 88016, 1616, -84784, -171184, -257584, -343984, -430384, -516784, -603184, -689584, -775984, -862384, 1616, 0 ] }, { "tz" : "America/Ojinaga", "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ], @@ -724,8 +728,8 @@ "diffs" : [ 172160, 85760, -640, -87040, -173440, -259840, -346240, -432640, -519040, -605440, -691840, -778240, -864640, -640, -660, 0 ] }, { "tz" : "America/Port_of_Spain", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Porto_Acre", "switches" : [ -62135751600, -59006430000, -55850670000, -52694910000, -46383390000, -43227630000, -40071870000, -33760350000, -30604590000, -27448830000, -21137310000, -17981550000, -14825790000, -12219274800, -2208988800 ], @@ -740,8 +744,8 @@ "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Punta_Arenas", - "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2524510746, -2208988800 ], - "diffs" : [ 179020, 92620, 6220, -80180, -166580, -252980, -339380, -425780, -512180, -598580, -684980, -771380, -857780, 6220, 6166, 0 ] + "switches" : [ -62135758800, -59006437200, -55850677200, -52694917200, -46383397200, -43227637200, -40071877200, -33760357200, -30604597200, -27448837200, -21137317200, -17981557200, -14825797200, -12219282000, -2524510745, -2208988800 ], + "diffs" : [ 179020, 92620, 6220, -80180, -166580, -252980, -339380, -425780, -512180, -598580, -684980, -771380, -857780, 6220, 6165, 0 ] }, { "tz" : "America/Rainy_River", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2366732504 ], @@ -781,7 +785,7 @@ }, { "tz" : "America/Santiago", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ] + "diffs" : [ 175365, 88965, 2565, -83835, -170235, -256635, -343035, -429435, -515835, -602235, -688635, -775035, -861435, 2565, 0 ] }, { "tz" : "America/Santo_Domingo", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2524507224, -2208988800 ], @@ -804,28 +808,28 @@ "diffs" : [ 86473, 73, -86327, -172727, -259127, -345527, -431927, -518327, -604727, -691127, -777527, -863927, -950327, -86327, 73, 0 ] }, { "tz" : "America/St_Barthelemy", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/St_Johns", "switches" : [ -62135757000, -59006435400, -55850675400, -52694915400, -46383395400, -43227635400, -40071875400, -33760355400, -30604595400, -27448835400, -21137315400, -17981555400, -14825795400, -12219280200, -2208988800 ], "diffs" : [ 172852, 86452, 52, -86348, -172748, -259148, -345548, -431948, -518348, -604748, -691148, -777548, -863948, 52, 0 ] }, { "tz" : "America/St_Kitts", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/St_Lucia", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/St_Thomas", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/St_Vincent", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Swift_Current", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ], @@ -852,20 +856,20 @@ "diffs" : [ 173852, 87452, 1052, -85348, -171748, -258148, -344548, -430948, -517348, -603748, -690148, -776548, -862948, 1052, 0 ] }, { "tz" : "America/Tortola", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Vancouver", "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2713880852 ], "diffs" : [ 173548, 87148, 748, -85652, -172052, -258452, -344852, -431252, -517652, -604052, -690452, -776852, -863252, 748, 0 ] }, { "tz" : "America/Virgin", - "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 173164, 86764, 364, -86036, -172436, -258836, -345236, -431636, -518036, -604436, -690836, -777236, -863636, 364, 0 ] + "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2233035335 ], + "diffs" : [ 174265, 87865, 1465, -84935, -171335, -257735, -344135, -430535, -516935, -603335, -689735, -776135, -862535, 1465, 0 ] }, { "tz" : "America/Whitehorse", - "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ], - "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ] + "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ], + "diffs" : [ 180012, 93612, 7212, -79188, -165588, -251988, -338388, -424788, -511188, -597588, -683988, -770388, -856788, 7212, 0 ] }, { "tz" : "America/Winnipeg", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2602258284 ], @@ -880,20 +884,20 @@ "diffs" : [ 147600, 61200, -25200, -111600, -198000, -284400, -370800, -457200, -543600, -630000, -716400, -802800, -889200, -25200, 0 ] }, { "tz" : "Antarctica/Casey", - "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ], - "diffs" : [ 201600, 115200, 28800, -57600, -144000, -230400, -316800, -403200, -489600, -576000, -662400, -748800, -835200, 28800, 0 ] + "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ], + "diffs" : [ 212400, 126000, 39600, -46800, -133200, -219600, -306000, -392400, -478800, -565200, -651600, -738000, -824400, 39600, 0 ] }, { "tz" : "Antarctica/Davis", "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ], "diffs" : [ 198000, 111600, 25200, -61200, -147600, -234000, -320400, -406800, -493200, -579600, -666000, -752400, -838800, 25200, 0 ] }, { "tz" : "Antarctica/DumontDUrville", - "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2208988800 ], - "diffs" : [ 208800, 122400, 36000, -50400, -136800, -223200, -309600, -396000, -482400, -568800, -655200, -741600, -828000, 36000, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ], + "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ] }, { "tz" : "Antarctica/Macquarie", - "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2214262800, -2208988800 ], - "diffs" : [ 212400, 126000, 39600, -46800, -133200, -219600, -306000, -392400, -478800, -565200, -651600, -738000, -824400, 39600, 3600, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2214259200 ], + "diffs" : [ 208800, 122400, 36000, -50400, -136800, -223200, -309600, -396000, -482400, -568800, -655200, -741600, -828000, 36000, 0 ] }, { "tz" : "Antarctica/Mawson", "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ], @@ -917,7 +921,7 @@ }, { "tz" : "Antarctica/Syowa", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], - "diffs" : [ 183600, 97200, 10800, -75600, -162000, -248400, -334800, -421200, -507600, -594000, -680400, -766800, -853200, 10800, 0 ] + "diffs" : [ 172388, 85988, -412, -86812, -173212, -259612, -346012, -432412, -518812, -605212, -691612, -778012, -864412, -412, 0 ] }, { "tz" : "Antarctica/Troll", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ], @@ -925,11 +929,11 @@ }, { "tz" : "Antarctica/Vostok", "switches" : [ -62135791200, -59006469600, -55850709600, -52694949600, -46383429600, -43227669600, -40071909600, -33760389600, -30604629600, -27448869600, -21137349600, -17981589600, -14825829600, -12219314400, -2208988800 ], - "diffs" : [ 194400, 108000, 21600, -64800, -151200, -237600, -324000, -410400, -496800, -583200, -669600, -756000, -842400, 21600, 0 ] + "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ] }, { "tz" : "Arctic/Longyearbyen", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ], - "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ], + "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ] }, { "tz" : "Asia/Aden", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], @@ -997,7 +1001,7 @@ }, { "tz" : "Asia/Brunei", "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ], - "diffs" : [ 174020, 87620, 1220, -85180, -171580, -257980, -344380, -430780, -517180, -603580, -689980, -776380, -862780, 1220, 0 ] + "diffs" : [ 175120, 88720, 2320, -84080, -170480, -256880, -343280, -429680, -516080, -602480, -688880, -775280, -861680, 2320, 0 ] }, { "tz" : "Asia/Calcutta", "switches" : [ -62135789400, -59006467800, -55850707800, -52694947800, -46383427800, -43227667800, -40071907800, -33760387800, -30604627800, -27448867800, -21137347800, -17981587800, -14825827800, -12219312600, -3645235808, -3155695330, -2208988800 ], @@ -1065,7 +1069,7 @@ }, { "tz" : "Asia/Ho_Chi_Minh", "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ], - "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ] + "diffs" : [ 172410, 86010, -390, -86790, -173190, -259590, -345990, -432390, -518790, -605190, -691590, -777990, -864390, -390, 0 ] }, { "tz" : "Asia/Hong_Kong", "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ], @@ -1133,7 +1137,7 @@ }, { "tz" : "Asia/Kuala_Lumpur", "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ], - "diffs" : [ 177194, 90794, 4394, -82006, -168406, -254806, -341206, -427606, -514006, -600406, -686806, -773206, -859606, 4394, 0 ] + "diffs" : [ 176675, 90275, 3875, -82525, -168925, -255325, -341725, -428125, -514525, -600925, -687325, -773725, -860125, 3875, 0 ] }, { "tz" : "Asia/Kuching", "switches" : [ -62135798400, -59006476800, -55850716800, -52694956800, -46383436800, -43227676800, -40071916800, -33760396800, -30604636800, -27448876800, -21137356800, -17981596800, -14825836800, -12219321600, -2208988800 ], @@ -1221,7 +1225,7 @@ }, { "tz" : "Asia/Saigon", "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ], - "diffs" : [ 172400, 86000, -400, -86800, -173200, -259600, -346000, -432400, -518800, -605200, -691600, -778000, -864400, -400, 0 ] + "diffs" : [ 172410, 86010, -390, -86790, -173190, -259590, -345990, -432390, -518790, -605190, -691590, -777990, -864390, -390, 0 ] }, { "tz" : "Asia/Sakhalin", "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ], @@ -1352,8 +1356,8 @@ "diffs" : [ 174424, 88024, 1624, -84776, -171176, -257576, -343976, -430376, -516776, -603176, -689576, -775976, -862376, 1624, 0 ] }, { "tz" : "Atlantic/Jan_Mayen", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ], - "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ], + "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ] }, { "tz" : "Atlantic/Madeira", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -1361,7 +1365,7 @@ }, { "tz" : "Atlantic/Reykjavik", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], - "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ] + "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ] }, { "tz" : "Atlantic/South_Georgia", "switches" : [ -62135762400, -59006440800, -55850680800, -52694920800, -46383400800, -43227640800, -40071880800, -33760360800, -30604600800, -27448840800, -21137320800, -17981560800, -14825800800, -12219285600, -2524512832 ], @@ -1396,8 +1400,8 @@ "diffs" : [ 172508, 86108, -292, -86692, -173092, -259492, -345892, -432292, -518692, -605092, -691492, -777892, -864292, -292, 0 ] }, { "tz" : "Australia/Currie", - "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345794528 ], - "diffs" : [ 174272, 87872, 1472, -84928, -171328, -257728, -344128, -430528, -516928, -603328, -689728, -776128, -862528, 1472, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2345795356 ], + "diffs" : [ 173444, 87044, 644, -85756, -172156, -258556, -344956, -431356, -517756, -604156, -690556, -776956, -863356, 644, 0 ] }, { "tz" : "Australia/Darwin", "switches" : [ -62135803800, -59006482200, -55850722200, -52694962200, -46383442200, -43227682200, -40071922200, -33760402200, -30604642200, -27448882200, -21137362200, -17981602200, -14825842200, -12219327000, -2364110000, -2230189200 ], @@ -1520,12 +1524,12 @@ "diffs" : [ 176316, 89916, 3516, -82884, -169284, -255684, -342084, -428484, -514884, -601284, -687684, -774084, -860484, 3516, 0 ] }, { "tz" : "Canada/Yukon", - "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2208988800 ], - "diffs" : [ 176412, 90012, 3612, -82788, -169188, -255588, -341988, -428388, -514788, -601188, -687588, -773988, -860388, 3612, 0 ] + "switches" : [ -62135744400, -59006422800, -55850662800, -52694902800, -46383382800, -43227622800, -40071862800, -33760342800, -30604582800, -27448822800, -21137302800, -17981542800, -14825782800, -12219267600, -2208988800 ], + "diffs" : [ 180012, 93612, 7212, -79188, -165588, -251988, -338388, -424788, -511188, -597588, -683988, -770388, -856788, 7212, 0 ] }, { "tz" : "Chile/Continental", "switches" : [ -62135755200, -59006433600, -55850673600, -52694913600, -46383393600, -43227633600, -40071873600, -33760353600, -30604593600, -27448833600, -21137313600, -17981553600, -14825793600, -12219278400, -2208988800 ], - "diffs" : [ 175366, 88966, 2566, -83834, -170234, -256634, -343034, -429434, -515834, -602234, -688634, -775034, -861434, 2566, 0 ] + "diffs" : [ 175365, 88965, 2565, -83835, -170235, -256635, -343035, -429435, -515835, -602235, -688635, -775035, -861435, 2565, 0 ] }, { "tz" : "Chile/EasterIsland", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ], @@ -1548,8 +1552,8 @@ "diffs" : [ 172491, 86091, -309, -86709, -173109, -259509, -345909, -432309, -518709, -605109, -691509, -777909, -864309, -309, 0 ] }, { "tz" : "Eire", - "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ], - "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ] + "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], + "diffs" : [ 174321, 87921, 1521, -84879, -171279, -257679, -344079, -430479, -516879, -603279, -689679, -776079, -862479, 1521, 0 ] }, { "tz" : "Etc/GMT", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800 ], @@ -1692,8 +1696,8 @@ "diffs" : [ 172800, 86400, 0, -86400, -172800, -259200, -345600, -432000, -518400, -604800, -691200, -777600, -864000, 0 ] }, { "tz" : "Europe/Amsterdam", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 175228, 88828, 2428, -83972, -170372, -256772, -343172, -429572, -515972, -602372, -688772, -775172, -861572, 2428, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2450998800, -2208988800 ], + "diffs" : [ 175350, 88950, 2550, -83850, -170250, -256650, -343050, -429450, -515850, -602250, -688650, -775050, -861450, 2550, 3600, 0 ] }, { "tz" : "Europe/Andorra", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -1732,7 +1736,7 @@ "diffs" : [ 173736, 87336, 936, -85464, -171864, -258264, -344664, -431064, -517464, -603864, -690264, -776664, -863064, 936, 0 ] }, { "tz" : "Europe/Budapest", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2500938980 ], + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2498260580 ], "diffs" : [ 171820, 85420, -980, -87380, -173780, -260180, -346580, -432980, -519380, -605780, -692180, -778580, -864980, -980, 0 ] }, { "tz" : "Europe/Busingen", @@ -1744,12 +1748,12 @@ "diffs" : [ 173080, 86680, 280, -86120, -172520, -258920, -345320, -431720, -518120, -604520, -690920, -777320, -863720, 280, 300, 0 ] }, { "tz" : "Europe/Copenhagen", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2398294220 ], - "diffs" : [ 173380, 86980, 580, -85820, -172220, -258620, -345020, -431420, -517820, -604220, -690620, -777020, -863420, 580, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ], + "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ] }, { "tz" : "Europe/Dublin", - "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2821651221, -2208988800 ], - "diffs" : [ 174300, 87900, 1500, -84900, -171300, -257700, -344100, -430500, -516900, -603300, -689700, -776100, -862500, 1500, 1521, 0 ] + "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], + "diffs" : [ 174321, 87921, 1521, -84879, -171279, -257679, -344079, -430479, -516879, -603279, -689679, -776079, -862479, 1521, 0 ] }, { "tz" : "Europe/Gibraltar", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2821653516, -2208988800 ], @@ -1786,6 +1790,10 @@ "tz" : "Europe/Kirov", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], "diffs" : [ 171672, 85272, -1128, -87528, -173928, -260328, -346728, -433128, -519528, -605928, -692328, -778728, -865128, -1128, 0 ] +}, { + "tz" : "Europe/Kyiv", + "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], + "diffs" : [ 172676, 86276, -124, -86524, -172924, -259324, -345724, -432124, -518524, -604924, -691324, -777724, -864124, -124, 0 ] }, { "tz" : "Europe/Lisbon", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], @@ -1800,8 +1808,8 @@ "diffs" : [ 172875, 86475, 75, -86325, -172725, -259125, -345525, -431925, -518325, -604725, -691125, -777525, -863925, 75, 0 ] }, { "tz" : "Europe/Luxembourg", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], - "diffs" : [ 174924, 88524, 2124, -84276, -170676, -257076, -343476, -429876, -516276, -602676, -689076, -775476, -861876, 2124, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2450998800, -2208988800 ], + "diffs" : [ 175350, 88950, 2550, -83850, -170250, -256650, -343050, -429450, -515850, -602250, -688650, -775050, -861450, 2550, 3600, 0 ] }, { "tz" : "Europe/Madrid", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -1820,8 +1828,8 @@ "diffs" : [ 176984, 90584, 4184, -82216, -168616, -255016, -341416, -427816, -514216, -600616, -687016, -773416, -859816, 4184, 4200, 0 ] }, { "tz" : "Europe/Monaco", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2486683211, -2208988800 ], - "diffs" : [ 174628, 88228, 1828, -84572, -170972, -257372, -343772, -430172, -516572, -602972, -689372, -775772, -862172, 1828, 3039, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], + "diffs" : [ 175839, 89439, 3039, -83361, -169761, -256161, -342561, -428961, -515361, -601761, -688161, -774561, -860961, 3039, 0 ] }, { "tz" : "Europe/Moscow", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], @@ -1832,8 +1840,8 @@ "diffs" : [ 171992, 85592, -808, -87208, -173608, -260008, -346408, -432808, -519208, -605608, -692008, -778408, -864808, -808, 0 ] }, { "tz" : "Europe/Oslo", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2366757780 ], - "diffs" : [ 173820, 87420, 1020, -85380, -171780, -258180, -344580, -430980, -517380, -603780, -690180, -776580, -862980, 1020, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ], + "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ] }, { "tz" : "Europe/Paris", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -1884,8 +1892,8 @@ "diffs" : [ 174404, 88004, 1604, -84796, -171196, -257596, -343996, -430396, -516796, -603196, -689596, -775996, -862396, 1604, 184, 0 ] }, { "tz" : "Europe/Stockholm", - "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2871681118, -2208992414 ], - "diffs" : [ 172068, 85668, -732, -87132, -173532, -259932, -346332, -432732, -519132, -605532, -691932, -778332, -864732, -732, -14, 0 ] + "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2422054408 ], + "diffs" : [ 173192, 86792, 392, -86008, -172408, -258808, -345208, -431608, -518008, -604408, -690808, -777208, -863608, 392, 0 ] }, { "tz" : "Europe/Tallinn", "switches" : [ -62135776800, -59006455200, -55850695200, -52694935200, -46383415200, -43227655200, -40071895200, -33760375200, -30604615200, -27448855200, -21137335200, -17981575200, -14825815200, -12219300000, -2208988800 ], @@ -1924,8 +1932,8 @@ "diffs" : [ 173924, 87524, 1124, -85276, -171676, -258076, -344476, -430876, -517276, -603676, -690076, -776476, -862876, 1124, 2160, 0 ] }, { "tz" : "Europe/Volgograd", - "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ], - "diffs" : [ 176540, 90140, 3740, -82660, -169060, -255460, -341860, -428260, -514660, -601060, -687460, -773860, -860260, 3740, 0 ] + "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], + "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 0 ] }, { "tz" : "Europe/Warsaw", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -1969,7 +1977,7 @@ }, { "tz" : "Iceland", "switches" : [ -62135769600, -59006448000, -55850688000, -52694928000, -46383408000, -43227648000, -40071888000, -33760368000, -30604608000, -27448848000, -21137328000, -17981568000, -14825808000, -12219292800, -2208988800 ], - "diffs" : [ 178080, 91680, 5280, -81120, -167520, -253920, -340320, -426720, -513120, -599520, -685920, -772320, -858720, 5280, 0 ] + "diffs" : [ 173768, 87368, 968, -85432, -171832, -258232, -344632, -431032, -517432, -603832, -690232, -776632, -863032, 968, 0 ] }, { "tz" : "Indian/Antananarivo", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], @@ -1980,12 +1988,12 @@ "diffs" : [ 177020, 90620, 4220, -82180, -168580, -254980, -341380, -427780, -514180, -600580, -686980, -773380, -859780, 4220, 0 ] }, { "tz" : "Indian/Christmas", - "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2364102172 ], - "diffs" : [ 172628, 86228, -172, -86572, -172972, -259372, -345772, -432172, -518572, -604972, -691372, -777772, -864172, -172, 0 ] + "switches" : [ -62135794800, -59006473200, -55850713200, -52694953200, -46383433200, -43227673200, -40071913200, -33760393200, -30604633200, -27448873200, -21137353200, -17981593200, -14825833200, -12219318000, -2208988800 ], + "diffs" : [ 173876, 87476, 1076, -85324, -171724, -258124, -344524, -430924, -517324, -603724, -690124, -776524, -862924, 1076, 0 ] }, { "tz" : "Indian/Cocos", - "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2209012060 ], - "diffs" : [ 172940, 86540, 140, -86260, -172660, -259060, -345460, -431860, -518260, -604660, -691060, -777460, -863860, 140, 0 ] + "switches" : [ -62135793000, -59006471400, -55850711400, -52694951400, -46383431400, -43227671400, -40071911400, -33760391400, -30604631400, -27448871400, -21137351400, -17981591400, -14825831400, -12219316200, -2208988800 ], + "diffs" : [ 173113, 86713, 313, -86087, -172487, -258887, -345287, -431687, -518087, -604487, -690887, -777287, -863687, 313, 0 ] }, { "tz" : "Indian/Comoro", "switches" : [ -62135780400, -59006458800, -55850698800, -52694938800, -46383418800, -43227658800, -40071898800, -33760378800, -30604618800, -27448858800, -21137338800, -17981578800, -14825818800, -12219303600, -2208988800 ], @@ -1993,11 +2001,11 @@ }, { "tz" : "Indian/Kerguelen", "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ], - "diffs" : [ 190800, 104400, 18000, -68400, -154800, -241200, -327600, -414000, -500400, -586800, -673200, -759600, -846000, 18000, 0 ] + "diffs" : [ 173160, 86760, 360, -86040, -172440, -258840, -345240, -431640, -518040, -604440, -690840, -777240, -863640, 360, 0 ] }, { "tz" : "Indian/Mahe", "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ], - "diffs" : [ 173892, 87492, 1092, -85308, -171708, -258108, -344508, -430908, -517308, -603708, -690108, -776508, -862908, 1092, 0 ] + "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ] }, { "tz" : "Indian/Maldives", "switches" : [ -62135787600, -59006466000, -55850706000, -52694946000, -46383426000, -43227666000, -40071906000, -33760386000, -30604626000, -27448866000, -21137346000, -17981586000, -14825826000, -12219310800, -2208988800 ], @@ -2013,7 +2021,7 @@ }, { "tz" : "Indian/Reunion", "switches" : [ -62135784000, -59006462400, -55850702400, -52694942400, -46383422400, -43227662400, -40071902400, -33760382400, -30604622400, -27448862400, -21137342400, -17981582400, -14825822400, -12219307200, -2208988800 ], - "diffs" : [ 173888, 87488, 1088, -85312, -171712, -258112, -344512, -430912, -517312, -603712, -690112, -776512, -862912, 1088, 0 ] + "diffs" : [ 173928, 87528, 1128, -85272, -171672, -258072, -344472, -430872, -517272, -603672, -690072, -776472, -862872, 1128, 0 ] }, { "tz" : "Iran", "switches" : [ -62135782200, -59006460600, -55850700600, -52694940600, -46383420600, -43227660600, -40071900600, -33760380600, -30604620600, -27448860600, -21137340600, -17981580600, -14825820600, -12219305400, -2208988800 ], @@ -2096,8 +2104,8 @@ "diffs" : [ 174672, 88272, 1872, -84528, -170928, -257328, -343728, -430128, -516528, -602928, -689328, -775728, -862128, 1872, 1800, 0 ] }, { "tz" : "Pacific/Chuuk", - "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ], - "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ], + "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ] }, { "tz" : "Pacific/Easter", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ], @@ -2109,7 +2117,7 @@ }, { "tz" : "Pacific/Enderbury", "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ], - "diffs" : [ 260660, 174260, 87860, 1460, -84940, -171340, -257740, -344140, -430540, -516940, -603340, -689740, -776140, 87860, 0 ] + "diffs" : [ 219600, 133200, 46800, -39600, -126000, -212400, -298800, -385200, -471600, -558000, -644400, -730800, -817200, 46800, 0 ] }, { "tz" : "Pacific/Fakaofo", "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ], @@ -2121,7 +2129,7 @@ }, { "tz" : "Pacific/Funafuti", "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ], - "diffs" : [ 172988, 86588, 188, -86212, -172612, -259012, -345412, -431812, -518212, -604612, -691012, -777412, -863812, 188, 0 ] + "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ] }, { "tz" : "Pacific/Galapagos", "switches" : [ -62135748000, -59006426400, -55850666400, -52694906400, -46383386400, -43227626400, -40071866400, -33760346400, -30604586400, -27448826400, -21137306400, -17981546400, -14825786400, -12219271200, -2208988800 ], @@ -2146,6 +2154,10 @@ "tz" : "Pacific/Johnston", "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2334103114, -2208988800 ], "diffs" : [ 174686, 88286, 1886, -84514, -170914, -257314, -343714, -430114, -516514, -602914, -689314, -775714, -862114, 1886, 1800, 0 ] +}, { + "tz" : "Pacific/Kanton", + "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ], + "diffs" : [ 219600, 133200, 46800, -39600, -126000, -212400, -298800, -385200, -471600, -558000, -644400, -730800, -817200, 46800, 0 ] }, { "tz" : "Pacific/Kiritimati", "switches" : [ -62135820000, -59006498400, -55850738400, -52694978400, -46383458400, -43227698400, -40071938400, -33760418400, -30604658400, -27448898400, -21137378400, -17981618400, -14825858400, -12219343200, -2208988800 ], @@ -2161,7 +2173,7 @@ }, { "tz" : "Pacific/Majuro", "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ], - "diffs" : [ 174912, 88512, 2112, -84288, -170688, -257088, -343488, -429888, -516288, -602688, -689088, -775488, -861888, 2112, 0 ] + "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ] }, { "tz" : "Pacific/Marquesas", "switches" : [ -62135735400, -59006413800, -55850653800, -52694893800, -46383373800, -43227613800, -40071853800, -33760333800, -30604573800, -27448813800, -21137293800, -17981533800, -14825773800, -12219258600, -2208988800 ], @@ -2200,20 +2212,20 @@ "diffs" : [ 175220, 88820, 2420, -83980, -170380, -256780, -343180, -429580, -515980, -602380, -688780, -775180, -861580, 2420, 0 ] }, { "tz" : "Pacific/Pohnpei", - "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ], - "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ] + "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ], + "diffs" : [ 174012, 87612, 1212, -85188, -171588, -257988, -344388, -430788, -517188, -603588, -689988, -776388, -862788, 1212, 0 ] }, { "tz" : "Pacific/Ponape", - "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -3944631600, -2208988800 ], - "diffs" : [ 260828, 174428, 88028, 1628, -84772, -171172, -257572, -343972, -430372, -516772, -603172, -689572, -775972, 88028, 1628, 0 ] + "switches" : [ -62135809200, -59006487600, -55850727600, -52694967600, -46383447600, -43227687600, -40071927600, -33760407600, -30604647600, -27448887600, -21137367600, -17981607600, -14825847600, -12219332400, -2208988800 ], + "diffs" : [ 174012, 87612, 1212, -85188, -171588, -257988, -344388, -430788, -517188, -603588, -689988, -776388, -862788, 1212, 0 ] }, { "tz" : "Pacific/Port_Moresby", "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ], "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ] }, { "tz" : "Pacific/Rarotonga", - "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2208988800 ], - "diffs" : [ 175144, 88744, 2344, -84056, -170456, -256856, -343256, -429656, -516056, -602456, -688856, -775256, -861656, 2344, 0 ] + "switches" : [ -62135733600, -59006412000, -55850652000, -52694892000, -46383372000, -43227612000, -40071852000, -33760332000, -30604572000, -27448812000, -21137292000, -17981532000, -14825772000, -12219256800, -2209557600, -2208988800 ], + "diffs" : [ 88744, 2344, -84056, -170456, -256856, -343256, -429656, -516056, -602456, -688856, -775256, -861656, -948056, -84056, 2344, 0 ] }, { "tz" : "Pacific/Saipan", "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ], @@ -2233,23 +2245,23 @@ }, { "tz" : "Pacific/Tongatapu", "switches" : [ -62135816400, -59006494800, -55850734800, -52694974800, -46383454800, -43227694800, -40071934800, -33760414800, -30604654800, -27448894800, -21137374800, -17981614800, -14825854800, -12219339600, -2208988800 ], - "diffs" : [ 175240, 88840, 2440, -83960, -170360, -256760, -343160, -429560, -515960, -602360, -688760, -775160, -861560, 2440, 0 ] + "diffs" : [ 175248, 88848, 2448, -83952, -170352, -256752, -343152, -429552, -515952, -602352, -688752, -775152, -861552, 2448, 0 ] }, { "tz" : "Pacific/Truk", - "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ], - "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ], + "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ] }, { "tz" : "Pacific/Wake", "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ], - "diffs" : [ 176012, 89612, 3212, -83188, -169588, -255988, -342388, -428788, -515188, -601588, -687988, -774388, -860788, 3212, 0 ] + "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ] }, { "tz" : "Pacific/Wallis", "switches" : [ -62135812800, -59006491200, -55850731200, -52694971200, -46383451200, -43227691200, -40071931200, -33760411200, -30604651200, -27448891200, -21137371200, -17981611200, -14825851200, -12219336000, -2208988800 ], - "diffs" : [ 171880, 85480, -920, -87320, -173720, -260120, -346520, -432920, -519320, -605720, -692120, -778520, -864920, -920, 0 ] + "diffs" : [ 174476, 88076, 1676, -84724, -171124, -257524, -343924, -430324, -516724, -603124, -689524, -775924, -862324, 1676, 0 ] }, { "tz" : "Pacific/Yap", - "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -3944628000, -2208988800 ], - "diffs" : [ 258772, 172372, 85972, -428, -86828, -173228, -259628, -346028, -432428, -518828, -605228, -691628, -778028, 85972, -428, 0 ] + "switches" : [ -62135805600, -59006484000, -55850724000, -52694964000, -46383444000, -43227684000, -40071924000, -33760404000, -30604644000, -27448884000, -21137364000, -17981604000, -14825844000, -12219328800, -2840176808, -2366790512 ], + "diffs" : [ 173480, 87080, 680, -85720, -172120, -258520, -344920, -431320, -517720, -604120, -690520, -776920, -863320, 680, 688, 0 ] }, { "tz" : "Poland", "switches" : [ -62135773200, -59006451600, -55850691600, -52694931600, -46383411600, -43227651600, -40071891600, -33760371600, -30604611600, -27448851600, -21137331600, -17981571600, -14825811600, -12219296400, -2208988800 ], @@ -2370,10 +2382,6 @@ "tz" : "US/Pacific", "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ], "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ] -}, { - "tz" : "US/Pacific-New", - "switches" : [ -62135740800, -59006419200, -55850659200, -52694899200, -46383379200, -43227619200, -40071859200, -33760339200, -30604579200, -27448819200, -21137299200, -17981539200, -14825779200, -12219264000, -2717640000 ], - "diffs" : [ 172378, 85978, -422, -86822, -173222, -259622, -346022, -432422, -518822, -605222, -691622, -778022, -864422, -422, 0 ] }, { "tz" : "US/Samoa", "switches" : [ -62135730000, -59006408400, -55850648400, -52694888400, -46383368400, -43227608400, -40071848400, -33760328400, -30604568400, -27448808400, -21137288400, -17981528400, -14825768400, -12219253200, -2445426000, -2208988800 ], diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BloomFilterMightContain.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BloomFilterMightContain.scala index 5cb19d36b805..b2273b6a6d13 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BloomFilterMightContain.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BloomFilterMightContain.scala @@ -76,8 +76,7 @@ case class BloomFilterMightContain( "functionName" -> toSQLId(prettyName), "expectedLeft" -> toSQLType(BinaryType), "expectedRight" -> toSQLType(LongType), - "actualLeft" -> toSQLType(left.dataType), - "actualRight" -> toSQLType(right.dataType) + "actual" -> Seq(left.dataType, right.dataType).map(toSQLType).mkString(", ") ) ) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/BloomFilterAggregate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/BloomFilterAggregate.scala index c734bca3ef8d..5b78c5b52286 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/BloomFilterAggregate.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/BloomFilterAggregate.scala @@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult._ import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLExpr, toSQLId, toSQLType, toSQLValue} import org.apache.spark.sql.catalyst.trees.TernaryLike import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ @@ -63,28 +64,66 @@ case class BloomFilterAggregate( override def checkInputDataTypes(): TypeCheckResult = { (first.dataType, second.dataType, third.dataType) match { case (_, NullType, _) | (_, _, NullType) => - TypeCheckResult.TypeCheckFailure("Null typed values cannot be used as size arguments") + DataTypeMismatch( + errorSubClass = "UNEXPECTED_NULL", + messageParameters = Map( + "exprName" -> "estimatedNumItems or numBits" + ) + ) case (LongType, LongType, LongType) => if (!estimatedNumItemsExpression.foldable) { - TypeCheckFailure("The estimated number of items provided must be a constant literal") + DataTypeMismatch( + errorSubClass = "NON_FOLDABLE_INPUT", + messageParameters = Map( + "inputName" -> "estimatedNumItems", + "inputType" -> toSQLType(estimatedNumItemsExpression.dataType), + "inputExpr" -> toSQLExpr(estimatedNumItemsExpression) + ) + ) } else if (estimatedNumItems <= 0L) { - TypeCheckFailure("The estimated number of items must be a positive value " + - s" (current value = $estimatedNumItems)") + DataTypeMismatch( + errorSubClass = "VALUE_OUT_OF_RANGE", + messageParameters = Map( + "exprName" -> "estimatedNumItems", + "valueRange" -> s"[0, positive]", + "currentValue" -> toSQLValue(estimatedNumItems, LongType) + ) + ) } else if (!numBitsExpression.foldable) { - TypeCheckFailure("The number of bits provided must be a constant literal") + DataTypeMismatch( + errorSubClass = "NON_FOLDABLE_INPUT", + messageParameters = Map( + "inputName" -> "numBitsExpression", + "inputType" -> toSQLType(numBitsExpression.dataType), + "inputExpr" -> toSQLExpr(numBitsExpression) + ) + ) } else if (numBits <= 0L) { - TypeCheckFailure("The number of bits must be a positive value " + - s" (current value = $numBits)") + DataTypeMismatch( + errorSubClass = "VALUE_OUT_OF_RANGE", + messageParameters = Map( + "exprName" -> "numBits", + "valueRange" -> s"[0, positive]", + "currentValue" -> toSQLValue(numBits, LongType) + ) + ) } else { require(estimatedNumItems <= SQLConf.get.getConf(SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS)) require(numBits <= SQLConf.get.getConf(SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_BITS)) TypeCheckSuccess } - case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " + - s"been a ${LongType.simpleString} value followed with two ${LongType.simpleString} size " + - s"arguments, but it's [${first.dataType.catalogString}, " + - s"${second.dataType.catalogString}, ${third.dataType.catalogString}]") + case _ => + DataTypeMismatch( + errorSubClass = "BLOOM_FILTER_WRONG_TYPE", + messageParameters = Map( + "functionName" -> toSQLId(prettyName), + "expectedLeft" -> toSQLType(BinaryType), + "expectedRight" -> toSQLType(LongType), + "actual" -> Seq(first.dataType, second.dataType, third.dataType) + .map(toSQLType).mkString(", ") + ) + ) } } override def nullable: Boolean = true diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 22a381278f5d..e8bad46e84a0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -3072,9 +3072,9 @@ object SubtractDates { """, examples = """ Examples: - > SELECT _FUNC_('Europe/Amsterdam', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); + > SELECT _FUNC_('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); 2021-12-05 15:00:00 - > SELECT _FUNC_('Europe/Amsterdam', timestamp_ntz'2021-12-05 15:00:00'); + > SELECT _FUNC_('Europe/Brussels', timestamp_ntz'2021-12-05 15:00:00'); 2021-12-06 00:00:00 """, group = "datetime_funcs", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index a99a795018d7..72eb420de374 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1924,7 +1924,7 @@ object SQLConf { "Integration Guide.") .version("3.1.0") .booleanConf - .createWithDefault(true) + .createWithDefault(false) val STATEFUL_OPERATOR_CHECK_CORRECTNESS_ENABLED = buildConf("spark.sql.streaming.statefulOperator.checkCorrectness.enabled") diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 26d5cf623874..e39d15189900 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1887,7 +1887,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation( ConvertTimezone( - Literal("Europe/Amsterdam"), + Literal("Europe/Brussels"), Literal("Europe/Moscow"), Literal(LocalDateTime.of(2022, 3, 27, 3, 0, 0))), LocalDateTime.of(2022, 3, 27, 4, 0, 0)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala index 66aef1b4b6cb..537f8d1f8060 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeTestUtils.scala @@ -49,9 +49,9 @@ object DateTimeTestUtils { CET.getId, "Africa/Dakar", LA.getId, - "Antarctica/Vostok", + "Asia/Urumqi", "Asia/Hong_Kong", - "Europe/Amsterdam") + "Europe/Brussels") val outstandingZoneIds: Seq[ZoneId] = outstandingTimezonesIds.map(getZoneId) def withDefaultTimeZone[T](newDefaultTimeZone: ZoneId)(block: => T): T = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 41da5409feb0..fdc17eed77b3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -937,16 +937,16 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { ("1970-01-01T00:00:00", "UTC") -> ("1969-12-31T16:00:00", "America/Los_Angeles"), ("2021-12-05T22:00:00", "Europe/Moscow") -> ("2021-12-06T00:00:00", "Asia/Yekaterinburg"), ("2021-12-06T00:01:02.123456", "Asia/Yekaterinburg") -> - ("2021-12-05T20:01:02.123456", "Europe/Amsterdam"), + ("2021-12-05T20:01:02.123456", "Europe/Brussels"), // 7 Nov 2021 is the DST day in the America/Los_Angeles time zone // Sunday, 7 November 2021, 02:00:00 clocks were turned backward 1 hour to // Sunday, 7 November 2021, 01:00:00 local standard time instead. - ("2021-11-07T09:00:00", "Europe/Amsterdam") -> ("2021-11-07T01:00:00", "America/Los_Angeles"), - ("2021-11-07T10:00:00", "Europe/Amsterdam") -> ("2021-11-07T01:00:00", "America/Los_Angeles"), - ("2021-11-07T11:00:00", "Europe/Amsterdam") -> ("2021-11-07T02:00:00", "America/Los_Angeles"), - ("2021-11-07T00:30:00", "America/Los_Angeles") -> ("2021-11-07T08:30:00", "Europe/Amsterdam"), - ("2021-11-07T01:30:00", "America/Los_Angeles") -> ("2021-11-07T09:30:00", "Europe/Amsterdam"), - ("2021-11-07T02:30:00", "America/Los_Angeles") -> ("2021-11-07T11:30:00", "Europe/Amsterdam") + ("2021-11-07T09:00:00", "Europe/Brussels") -> ("2021-11-07T01:00:00", "America/Los_Angeles"), + ("2021-11-07T10:00:00", "Europe/Brussels") -> ("2021-11-07T01:00:00", "America/Los_Angeles"), + ("2021-11-07T11:00:00", "Europe/Brussels") -> ("2021-11-07T02:00:00", "America/Los_Angeles"), + ("2021-11-07T00:30:00", "America/Los_Angeles") -> ("2021-11-07T08:30:00", "Europe/Brussels"), + ("2021-11-07T01:30:00", "America/Los_Angeles") -> ("2021-11-07T09:30:00", "Europe/Brussels"), + ("2021-11-07T02:30:00", "America/Los_Angeles") -> ("2021-11-07T11:30:00", "Europe/Brussels") ).foreach { case ((inputTs, sourceTz), (expectedTs, targetTz)) => val micros = DateTimeUtils.localDateTimeToMicros(LocalDateTime.parse(inputTs)) val result = DateTimeUtils.convertTimestampNtzToAnotherTz(sourceTz, targetTz, micros) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala index 0d3f681c9864..a17ca2358de2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/RebaseDateTimeSuite.scala @@ -171,7 +171,7 @@ class RebaseDateTimeSuite extends SparkFunSuite with Matchers with SQLHelper { test("SPARK-31328: rebasing overlapped timestamps during daylight saving time") { Seq( LA.getId -> Seq("2019-11-03T08:00:00Z", "2019-11-03T08:30:00Z", "2019-11-03T09:00:00Z"), - "Europe/Amsterdam" -> + "Europe/Brussels" -> Seq("2019-10-27T00:00:00Z", "2019-10-27T00:30:00Z", "2019-10-27T01:00:00Z") ).foreach { case (tz, ts) => withDefaultTimeZone(getZoneId(tz)) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala index 56e73ead53de..b305d5f5afa5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/TimestampFormatterSuite.scala @@ -44,9 +44,9 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite { CET.getId -> 1543741872001234L, "Africa/Dakar" -> 1543745472001234L, "America/Los_Angeles" -> 1543774272001234L, - "Antarctica/Vostok" -> 1543723872001234L, + "Asia/Urumqi" -> 1543723872001234L, "Asia/Hong_Kong" -> 1543716672001234L, - "Europe/Amsterdam" -> 1543741872001234L) + "Europe/Brussels" -> 1543741872001234L) outstandingTimezonesIds.foreach { zoneId => val formatter = TimestampFormatter( "yyyy-MM-dd'T'HH:mm:ss.SSSSSS", @@ -65,9 +65,9 @@ class TimestampFormatterSuite extends DatetimeFormatterSuite { CET.getId -> "2018-12-02 11:11:12.001234", "Africa/Dakar" -> "2018-12-02 10:11:12.001234", "America/Los_Angeles" -> "2018-12-02 02:11:12.001234", - "Antarctica/Vostok" -> "2018-12-02 16:11:12.001234", + "Asia/Urumqi" -> "2018-12-02 16:11:12.001234", "Asia/Hong_Kong" -> "2018-12-02 18:11:12.001234", - "Europe/Amsterdam" -> "2018-12-02 11:11:12.001234") + "Europe/Brussels" -> "2018-12-02 11:11:12.001234") outstandingTimezonesIds.foreach { zoneId => Seq( TimestampFormatter( diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 34964bc71025..4ce4f1225ce6 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -74,7 +74,7 @@ | org.apache.spark.sql.catalyst.expressions.ConcatWs | concat_ws | SELECT concat_ws(' ', 'Spark', 'SQL') | struct | | org.apache.spark.sql.catalyst.expressions.ContainsExpressionBuilder | contains | SELECT contains('Spark SQL', 'Spark') | struct | | org.apache.spark.sql.catalyst.expressions.Conv | conv | SELECT conv('100', 2, 10) | struct | -| org.apache.spark.sql.catalyst.expressions.ConvertTimezone | convert_timezone | SELECT convert_timezone('Europe/Amsterdam', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00') | struct | +| org.apache.spark.sql.catalyst.expressions.ConvertTimezone | convert_timezone | SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00') | struct | | org.apache.spark.sql.catalyst.expressions.Cos | cos | SELECT cos(0) | struct | | org.apache.spark.sql.catalyst.expressions.Cosh | cosh | SELECT cosh(0) | struct | | org.apache.spark.sql.catalyst.expressions.Cot | cot | SELECT cot(1) | struct | diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql index 88ce0baa8cd2..377b26c67a3e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz.sql @@ -15,4 +15,4 @@ SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 45.678); SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 45.678, 'CET'); SELECT make_timestamp_ltz(2021, 07, 11, 6, 30, 60.007); -SELECT convert_timezone('Europe/Amsterdam', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles'); +SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz.sql index b75604485cd8..d744c0c19b42 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz.sql @@ -17,7 +17,7 @@ SELECT make_timestamp_ntz(2021, 07, 11, 6, 30, 45.678, 'CET'); SELECT make_timestamp_ntz(2021, 07, 11, 6, 30, 60.007); SELECT convert_timezone('Europe/Moscow', 'America/Los_Angeles', timestamp_ntz'2022-01-01 00:00:00'); -SELECT convert_timezone('Europe/Amsterdam', timestamp_ntz'2022-03-23 00:00:00'); +SELECT convert_timezone('Europe/Brussels', timestamp_ntz'2022-03-23 00:00:00'); -- Get the difference between timestamps w/o time zone in the specified units select timestampdiff(QUARTER, timestamp_ntz'2022-01-01 01:02:03', timestamp_ntz'2022-05-02 05:06:07'); diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out index 334632ca2bbb..b4ab5bdeb4ff 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz.sql.out @@ -56,8 +56,8 @@ NULL -- !query -SELECT convert_timezone('Europe/Amsterdam', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles') +SELECT convert_timezone('Europe/Brussels', timestamp_ltz'2022-03-23 00:00:00 America/Los_Angeles') -- !query schema -struct +struct -- !query output 2022-03-23 08:00:00 diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz.sql.out index dd0ea1ab687f..5a634f39a884 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz.sql.out @@ -79,9 +79,9 @@ struct +struct -- !query output 2022-03-23 08:00:00 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BloomFilterAggregateQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/BloomFilterAggregateQuerySuite.scala index 6a22414db00c..cf5d4c8c1e9a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/BloomFilterAggregateQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/BloomFilterAggregateQuerySuite.scala @@ -19,11 +19,13 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.Cast.toSQLValue import org.apache.spark.sql.catalyst.expressions.aggregate.BloomFilterAggregate import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.aggregate.BaseAggregateExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.LongType /** * Query tests for the Bloom filter aggregate and filter function. @@ -62,8 +64,8 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { val table = "bloom_filter_test" for (numEstimatedItems <- Seq(Long.MinValue, -10L, 0L, 4096L, 4194304L, Long.MaxValue, conf.getConf(SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_ITEMS))) { - for (numBits <- Seq(Long.MinValue, -10L, 0L, 4096L, 4194304L, Long.MaxValue, - conf.getConf(SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_BITS))) { + for ((numBits, index) <- Seq(Long.MinValue, -10L, 0L, 4096L, 4194304L, Long.MaxValue, + conf.getConf(SQLConf.RUNTIME_BLOOM_FILTER_MAX_NUM_BITS)).zipWithIndex) { val sqlString = s""" |SELECT every(might_contain( | (SELECT bloom_filter_agg(col, @@ -87,13 +89,57 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { val exception = intercept[AnalysisException] { spark.sql(sqlString) } - assert(exception.getMessage.contains( - "The estimated number of items must be a positive value")) + val stop = numEstimatedItems match { + case Long.MinValue => Seq(169, 152, 150, 153, 156, 168, 157) + case -10L => Seq(152, 135, 133, 136, 139, 151, 140) + case 0L => Seq(150, 133, 131, 134, 137, 149, 138) + } + checkError( + exception = exception, + errorClass = "DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE", + parameters = Map( + "exprName" -> "estimatedNumItems", + "valueRange" -> "[0, positive]", + "currentValue" -> toSQLValue(numEstimatedItems, LongType), + "sqlExpr" -> (s""""bloom_filter_agg(col, CAST($numEstimatedItems AS BIGINT), """ + + s"""CAST($numBits AS BIGINT))"""") + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(col,\n" + + s" cast($numEstimatedItems as long),\n" + + s" cast($numBits as long))", + start = 49, + stop = stop(index) + ) + ) } else if (numBits <= 0) { val exception = intercept[AnalysisException] { spark.sql(sqlString) } - assert(exception.getMessage.contains("The number of bits must be a positive value")) + val stop = numEstimatedItems match { + case 4096L => Seq(153, 136, 134) + case 4194304L => Seq(156, 139, 137) + case Long.MaxValue => Seq(168, 151, 149) + case 4000000 => Seq(156, 139, 137) + } + checkError( + exception = exception, + errorClass = "DATATYPE_MISMATCH.VALUE_OUT_OF_RANGE", + parameters = Map( + "exprName" -> "numBits", + "valueRange" -> "[0, positive]", + "currentValue" -> toSQLValue(numBits, LongType), + "sqlExpr" -> (s""""bloom_filter_agg(col, CAST($numEstimatedItems AS BIGINT), """ + + s"""CAST($numBits AS BIGINT))"""") + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(col,\n" + + s" cast($numEstimatedItems as long),\n" + + s" cast($numBits as long))", + start = 49, + stop = stop(index) + ) + ) } else { checkAnswer(spark.sql(sqlString), Row(true, false)) } @@ -109,8 +155,22 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { |FROM values (1.2), (2.5) as t(a)""" .stripMargin) } - assert(exception1.getMessage.contains( - "Input to function bloom_filter_agg should have been a bigint value")) + checkError( + exception = exception1, + errorClass = "DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE", + parameters = Map( + "functionName" -> "`bloom_filter_agg`", + "sqlExpr" -> "\"bloom_filter_agg(a, 1000000, 8388608)\"", + "expectedLeft" -> "\"BINARY\"", + "expectedRight" -> "\"BIGINT\"", + "actual" -> "\"DECIMAL(2,1)\", \"BIGINT\", \"BIGINT\"" + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(a)", + start = 8, + stop = 26 + ) + ) val exception2 = intercept[AnalysisException] { spark.sql(""" @@ -118,8 +178,22 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { |FROM values (cast(1 as long)), (cast(2 as long)) as t(a)""" .stripMargin) } - assert(exception2.getMessage.contains( - "function bloom_filter_agg should have been a bigint value followed with two bigint")) + checkError( + exception = exception2, + errorClass = "DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE", + parameters = Map( + "functionName" -> "`bloom_filter_agg`", + "sqlExpr" -> "\"bloom_filter_agg(a, 2, (2 * 8))\"", + "expectedLeft" -> "\"BINARY\"", + "expectedRight" -> "\"BIGINT\"", + "actual" -> "\"BIGINT\", \"INT\", \"BIGINT\"" + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(a, 2)", + start = 8, + stop = 29 + ) + ) val exception3 = intercept[AnalysisException] { spark.sql(""" @@ -127,8 +201,22 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { |FROM values (cast(1 as long)), (cast(2 as long)) as t(a)""" .stripMargin) } - assert(exception3.getMessage.contains( - "function bloom_filter_agg should have been a bigint value followed with two bigint")) + checkError( + exception = exception3, + errorClass = "DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE", + parameters = Map( + "functionName" -> "`bloom_filter_agg`", + "sqlExpr" -> "\"bloom_filter_agg(a, CAST(2 AS BIGINT), 5)\"", + "expectedLeft" -> "\"BINARY\"", + "expectedRight" -> "\"BIGINT\"", + "actual" -> "\"BIGINT\", \"BIGINT\", \"INT\"" + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(a, cast(2 as long), 5)", + start = 8, + stop = 46 + ) + ) val exception4 = intercept[AnalysisException] { spark.sql(""" @@ -136,7 +224,19 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { |FROM values (cast(1 as long)), (cast(2 as long)) as t(a)""" .stripMargin) } - assert(exception4.getMessage.contains("Null typed values cannot be used as size arguments")) + checkError( + exception = exception4, + errorClass = "DATATYPE_MISMATCH.UNEXPECTED_NULL", + parameters = Map( + "exprName" -> "estimatedNumItems or numBits", + "sqlExpr" -> "\"bloom_filter_agg(a, NULL, 5)\"" + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(a, null, 5)", + start = 8, + stop = 35 + ) + ) val exception5 = intercept[AnalysisException] { spark.sql(""" @@ -144,7 +244,19 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { |FROM values (cast(1 as long)), (cast(2 as long)) as t(a)""" .stripMargin) } - assert(exception5.getMessage.contains("Null typed values cannot be used as size arguments")) + checkError( + exception = exception5, + errorClass = "DATATYPE_MISMATCH.UNEXPECTED_NULL", + parameters = Map( + "exprName" -> "estimatedNumItems or numBits", + "sqlExpr" -> "\"bloom_filter_agg(a, 5, NULL)\"" + ), + context = ExpectedContext( + fragment = "bloom_filter_agg(a, 5, null)", + start = 8, + stop = 35 + ) + ) } test("Test that might_contain errors out disallowed input value types") { @@ -160,8 +272,7 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { "functionName" -> "`might_contain`", "expectedLeft" -> "\"BINARY\"", "expectedRight" -> "\"BIGINT\"", - "actualLeft" -> "\"DECIMAL(2,1)\"", - "actualRight" -> "\"BIGINT\"" + "actual" -> "\"DECIMAL(2,1)\", \"BIGINT\"" ), context = ExpectedContext( fragment = "might_contain(1.0, 1L)", @@ -182,8 +293,7 @@ class BloomFilterAggregateQuerySuite extends QueryTest with SharedSparkSession { "functionName" -> "`might_contain`", "expectedLeft" -> "\"BINARY\"", "expectedRight" -> "\"BIGINT\"", - "actualLeft" -> "\"VOID\"", - "actualRight" -> "\"DECIMAL(1,1)\"" + "actual" -> "\"VOID\", \"DECIMAL(1,1)\"" ), context = ExpectedContext( fragment = "might_contain(NULL, 0.1)", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala index febdb8f81bf5..ecb4bfd0ec41 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala @@ -2462,11 +2462,6 @@ class SubquerySuite extends QueryTest |create temp view v1(x) as values |from_json('{"a":1, "b":2}', 'map') t(x) |""".stripMargin) - sql( - """ - |create temp view v2(x) as values - |from_json('{"b":0, "c":2}', 'map') t(x) - |""".stripMargin) // Can use non-orderable data type in one row subquery that can be collapsed. checkAnswer( @@ -2475,7 +2470,7 @@ class SubquerySuite extends QueryTest // Cannot use non-orderable data type in one row subquery that cannot be collapsed. val error = intercept[AnalysisException] { - sql("select (select a + a from (select upper(x['a']) as a)) from v1").collect() + sql("select (select concat(a, a) from (select upper(x['a']) as a)) from v1").collect() } assert(error.getMessage.contains("Correlated column reference 'v1.x' cannot be map type")) } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index f6b06b08cbc7..213d930653dd 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -537,12 +537,18 @@ private[hive] class HiveClientImpl( storage = CatalogStorageFormat( locationUri = shim.getDataLocation(h).map { loc => val tableUri = stringToURI(loc) - // Before SPARK-19257, created data source table does not use absolute uri. - // This makes Spark can't read these tables across HDFS clusters. - // Rewrite table location to absolute uri based on database uri to fix this issue. - val absoluteUri = Option(tableUri).filterNot(_.isAbsolute) - .map(_ => stringToURI(client.getDatabase(h.getDbName).getLocationUri)) - HiveExternalCatalog.toAbsoluteURI(tableUri, absoluteUri) + if (h.getTableType == HiveTableType.VIRTUAL_VIEW) { + // Data location of SQL view is useless. Do not qualify it even if it's present, as + // it can be an invalid path. + tableUri + } else { + // Before SPARK-19257, created data source table does not use absolute uri. + // This makes Spark can't read these tables across HDFS clusters. + // Rewrite table location to absolute uri based on database uri to fix this issue. + val absoluteUri = Option(tableUri).filterNot(_.isAbsolute) + .map(_ => stringToURI(client.getDatabase(h.getDbName).getLocationUri)) + HiveExternalCatalog.toAbsoluteURI(tableUri, absoluteUri) + } }, // To avoid ClassNotFound exception, we try our best to not get the format class, but get // the class name directly. However, for non-native tables, there is no interface to get