diff --git a/docs/sql-data-sources-json.md b/docs/sql-data-sources-json.md index 0f1ca432b704..041512918e61 100644 --- a/docs/sql-data-sources-json.md +++ b/docs/sql-data-sources-json.md @@ -94,3 +94,168 @@ SELECT * FROM jsonTable + +## Data Source Option + +Data source options of JSON can be set via: +* the `.option`/`.options` methods of + * `DataFrameReader` + * `DataFrameWriter` + * `DataStreamReader` + * `DataStreamWriter` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Property NameDefaultMeaningScope
timeZoneNoneSets the string that indicates a time zone ID to be used to format timestamps in the JSON datasources or partition values. The following formats of timeZone are supported:
+
    +
  • Region-based zone ID: It should have the form 'area/city', such as 'America/Los_Angeles'.
    • +
  • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
    • +
+ Other short names like 'CST' are not recommended to use because they can be ambiguous. If it isn't set, the current value of the SQL config spark.sql.session.timeZone is used by default. + 		read/write
primitivesAsStringNoneInfers all primitive values as a string type. If None is set, it uses the default value, false.read
prefersDecimalNoneInfers all floating-point values as a decimal type. If the values do not fit in decimal, then it infers them as doubles. If None is set, it uses the default value, false.read
allowCommentsNoneIgnores Java/C++ style comment in JSON records. If None is set, it uses the default value, falseread
allowUnquotedFieldNamesNoneAllows unquoted JSON field names. If None is set, it uses the default value, false.read
allowSingleQuotesNoneAllows single quotes in addition to double quotes. If None is set, it uses the default value, true.read
allowNumericLeadingZeroNoneAllows leading zeros in numbers (e.g. 00012). If None is set, it uses the default value, false.read
allowBackslashEscapingAnyCharacterNoneAllows accepting quoting of all character using backslash quoting mechanism. If None is set, it uses the default value, false.read
modeNoneAllows a mode for dealing with corrupt records during parsing. If None is set, it uses the default value, PERMISSIVE
+
    +
  • PERMISSIVE: when it meets a corrupted record, puts the malformed string into a field configured by columnNameOfCorruptRecord, and sets malformed fields to null. To keep corrupt records, an user can set a string type field named columnNameOfCorruptRecord in an user-defined schema. If a schema does not have the field, it drops corrupt records during parsing. When inferring a schema, it implicitly adds a columnNameOfCorruptRecord field in an output schema.
    • +
  • DROPMALFORMED: ignores the whole corrupted records.
    • +
  • FAILFAST: throws an exception when it meets corrupted records.
    • +
+ 		read
columnNameOfCorruptRecordNoneAllows renaming the new field having malformed string created by PERMISSIVE mode. This overrides spark.sql.columnNameOfCorruptRecord. If None is set, it uses the value specified in spark.sql.columnNameOfCorruptRecord.read
dateFormatNoneSets the string that indicates a date format. Custom date formats follow the formats at datetime pattern. This applies to date type. If None is set, it uses the default value, yyyy-MM-dd.read/write
timestampFormatNoneSets the string that indicates a timestamp format. Custom date formats follow the formats at datetime pattern. This applies to timestamp type. If None is set, it uses the default value, yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX].read/write
multiLineNoneParse one record, which may span multiple lines, per file. If None is set, it uses the default value, false.read
allowUnquotedControlCharsNoneAllows JSON Strings to contain unquoted control characters (ASCII characters with value less than 32, including tab and line feed characters) or not.read
encodingNoneFor reading, allows to forcibly set one of standard basic or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE. If None is set, the encoding of input JSON will be detected automatically when the multiLine option is set to true. For writing, Specifies encoding (charset) of saved json files. If None is set, the default UTF-8 charset will be used.read/write
lineSepNoneDefines the line separator that should be used for parsing. If None is set, it covers all \r, \r\n and \n.read/write
samplingRatioNoneDefines fraction of input JSON objects used for schema inferring. If None is set, it uses the default value, 1.0.read
dropFieldIfAllNullNoneWhether to ignore column of all null values or empty array/struct during schema inference. If None is set, it uses the default value, false.read
localeNoneSets a locale as language tag in IETF BCP 47 format. If None is set, it uses the default value, en-US. For instance, locale is used while parsing dates and timestamps.read
allowNonNumericNumbersNoneAllows JSON parser to recognize set of “Not-a-Number” (NaN) tokens as legal floating number values. If None is set, it uses the default value, true.
+
    +
  • +INF: for positive infinity, as well as alias of +Infinity and Infinity.
    • +
  • -INF: for negative infinity, alias -Infinity.
    • +
  • NaN: for other not-a-numbers, like result of division by zero.
    • +
+ 		read
compressionNoneCompression codec to use when saving to file. This can be one of the known case-insensitive shorten names (none, bzip2, gzip, lz4, snappy and deflate).write
ignoreNullFieldsNoneWhether to ignore null fields when generating JSON objects. If None is set, it uses the default value, true.write
+Other generic options can be found in Generic File Source Options. diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 6ea73430367f..e013f4d4d3cf 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -3672,7 +3672,9 @@ def schema_of_json(json, options=None): json : :class:`~pyspark.sql.Column` or str a JSON string or a foldable string column containing a JSON string. options : dict, optional - options to control parsing. accepts the same options as the JSON datasource + options to control parsing. accepts the same options as the JSON datasource. + See `Data Source Option `_ # noqa + in the version you use. .. versionchanged:: 3.0 It accepts `options` parameter to control schema inferring. diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py index 31c1f2f7ca3c..9712a4bb5c93 100644 --- a/python/pyspark/sql/readwriter.py +++ b/python/pyspark/sql/readwriter.py @@ -108,29 +108,6 @@ def schema(self, schema): @since(1.5) def option(self, key, value): """Adds an input option for the underlying data source. - - You can set the following option(s) for reading files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to parse - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching - the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter. - It does not change the behavior of partition discovery. - * ``modifiedBefore``: an optional timestamp to only include files with - modification times occurring before the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) - * ``modifiedAfter``: an optional timestamp to only include files with - modification times occurring after the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) """ self._jreader = self._jreader.option(key, to_str(value)) return self @@ -138,29 +115,6 @@ def option(self, key, value): @since(1.4) def options(self, **options): """Adds input options for the underlying data source. - - You can set the following option(s) for reading files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to parse - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - * ``pathGlobFilter``: an optional glob pattern to only include files with paths matching - the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter. - It does not change the behavior of partition discovery. - * ``modifiedBefore``: an optional timestamp to only include files with - modification times occurring before the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) - * ``modifiedAfter``: an optional timestamp to only include files with - modification times occurring after the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) """ for k in options: self._jreader = self._jreader.option(k, to_str(options[k])) @@ -236,112 +190,13 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None, schema : :class:`pyspark.sql.types.StructType` or str, optional an optional :class:`pyspark.sql.types.StructType` for the input schema or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - primitivesAsString : str or bool, optional - infers all primitive values as a string type. If None is set, - it uses the default value, ``false``. - prefersDecimal : str or bool, optional - infers all floating-point values as a decimal type. If the values - do not fit in decimal, then it infers them as doubles. If None is - set, it uses the default value, ``false``. - allowComments : str or bool, optional - ignores Java/C++ style comment in JSON records. If None is set, - it uses the default value, ``false``. - allowUnquotedFieldNames : str or bool, optional - allows unquoted JSON field names. If None is set, - it uses the default value, ``false``. - allowSingleQuotes : str or bool, optional - allows single quotes in addition to double quotes. If None is - set, it uses the default value, ``true``. - allowNumericLeadingZero : str or bool, optional - allows leading zeros in numbers (e.g. 00012). If None is - set, it uses the default value, ``false``. - allowBackslashEscapingAnyCharacter : str or bool, optional - allows accepting quoting of all character - using backslash quoting mechanism. If None is - set, it uses the default value, ``false``. - mode : str, optional - allows a mode for dealing with corrupt records during parsing. If None is - set, it uses the default value, ``PERMISSIVE``. - - * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \ - into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \ - fields to ``null``. To keep corrupt records, an user can set a string type \ - field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \ - schema does not have the field, it drops corrupt records during parsing. \ - When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \ - field in an output schema. - * ``DROPMALFORMED``: ignores the whole corrupted records. - * ``FAILFAST``: throws an exception when it meets corrupted records. - - columnNameOfCorruptRecord: str, optional - allows renaming the new field having malformed string - created by ``PERMISSIVE`` mode. This overrides - ``spark.sql.columnNameOfCorruptRecord``. If None is set, - it uses the value specified in - ``spark.sql.columnNameOfCorruptRecord``. - dateFormat : str, optional - sets the string that indicates a date format. Custom date formats - follow the formats at - `datetime pattern `_. # noqa - This applies to date type. If None is set, it uses the - default value, ``yyyy-MM-dd``. - timestampFormat : str, optional - sets the string that indicates a timestamp format. - Custom date formats follow the formats at - `datetime pattern `_. # noqa - This applies to timestamp type. If None is set, it uses the - default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``. - multiLine : str or bool, optional - parse one record, which may span multiple lines, per file. If None is - set, it uses the default value, ``false``. - allowUnquotedControlChars : str or bool, optional - allows JSON Strings to contain unquoted control - characters (ASCII characters with value less than 32, - including tab and line feed characters) or not. - encoding : str or bool, optional - allows to forcibly set one of standard basic or extended encoding for - the JSON files. For example UTF-16BE, UTF-32LE. If None is set, - the encoding of input JSON will be detected automatically - when the multiLine option is set to ``true``. - lineSep : str, optional - defines the line separator that should be used for parsing. If None is - set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``. - samplingRatio : str or float, optional - defines fraction of input JSON objects used for schema inferring. - If None is set, it uses the default value, ``1.0``. - dropFieldIfAllNull : str or bool, optional - whether to ignore column of all null values or empty - array/struct during schema inference. If None is set, it - uses the default value, ``false``. - locale : str, optional - sets a locale as language tag in IETF BCP 47 format. If None is set, - it uses the default value, ``en-US``. For instance, ``locale`` is used while - parsing dates and timestamps. - pathGlobFilter : str or bool, optional - an optional glob pattern to only include files with paths matching - the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`. - It does not change the behavior of - `partition discovery `_. # noqa - recursiveFileLookup : str or bool, optional - recursively scan a directory for files. Using this option - disables - `partition discovery `_. # noqa - allowNonNumericNumbers : str or bool - allows JSON parser to recognize set of "Not-a-Number" (NaN) - tokens as legal floating number values. If None is set, - it uses the default value, ``true``. - - * ``+INF``: for positive infinity, as well as alias of - ``+Infinity`` and ``Infinity``. - * ``-INF``: for negative infinity, alias ``-Infinity``. - * ``NaN``: for other not-a-numbers, like result of division by zero. - modifiedBefore : an optional timestamp to only include files with - modification times occurring before the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) - modifiedAfter : an optional timestamp to only include files with - modification times occurring after the specified time. The provided timestamp - must be in the following format: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00) + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ # noqa + in the version you use. Examples -------- @@ -957,20 +812,6 @@ def format(self, source): @since(1.5) def option(self, key, value): """Adds an output option for the underlying data source. - - You can set the following option(s) for writing files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to format - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. """ self._jwrite = self._jwrite.option(key, to_str(value)) return self @@ -978,20 +819,6 @@ def option(self, key, value): @since(1.4) def options(self, **options): """Adds output options for the underlying data source. - - You can set the following option(s) for writing files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to format - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. """ for k in options: self._jwrite = self._jwrite.option(k, to_str(options[k])) @@ -1204,31 +1031,13 @@ def json(self, path, mode=None, compression=None, dateFormat=None, timestampForm * ``ignore``: Silently ignore this operation if data already exists. * ``error`` or ``errorifexists`` (default case): Throw an exception if data already \ exists. - compression : str, optional - compression codec to use when saving to file. This can be one of the - known case-insensitive shorten names (none, bzip2, gzip, lz4, - snappy and deflate). - dateFormat : str, optional - sets the string that indicates a date format. Custom date formats - follow the formats at - `datetime pattern `_. # noqa - This applies to date type. If None is set, it uses the - default value, ``yyyy-MM-dd``. - timestampFormat : str, optional - sets the string that indicates a timestamp format. - Custom date formats follow the formats at - `datetime pattern `_. # noqa - This applies to timestamp type. If None is set, it uses the - default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``. - encoding : str, optional - specifies encoding (charset) of saved json files. If None is set, - the default UTF-8 charset will be used. - lineSep : str, optional - defines the line separator that should be used for writing. If None is - set, it uses the default value, ``\\n``. - ignoreNullFields : str or bool, optional - Whether to ignore null fields when generating JSON objects. - If None is set, it uses the default value, ``true``. + + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ # noqa + in the version you use. Examples -------- diff --git a/python/pyspark/sql/streaming.py b/python/pyspark/sql/streaming.py index 2c90d7f2dee7..4a20dc7d7c13 100644 --- a/python/pyspark/sql/streaming.py +++ b/python/pyspark/sql/streaming.py @@ -378,20 +378,6 @@ def schema(self, schema): def option(self, key, value): """Adds an input option for the underlying data source. - You can set the following option(s) for reading files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to parse - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - .. versionadded:: 2.0.0 Notes @@ -408,20 +394,6 @@ def option(self, key, value): def options(self, **options): """Adds input options for the underlying data source. - You can set the following option(s) for reading files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to parse - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - .. versionadded:: 2.0.0 Notes @@ -507,102 +479,13 @@ def json(self, path, schema=None, primitivesAsString=None, prefersDecimal=None, schema : :class:`pyspark.sql.types.StructType` or str, optional an optional :class:`pyspark.sql.types.StructType` for the input schema or a DDL-formatted string (For example ``col0 INT, col1 DOUBLE``). - primitivesAsString : str or bool, optional - infers all primitive values as a string type. If None is set, - it uses the default value, ``false``. - prefersDecimal : str or bool, optional - infers all floating-point values as a decimal type. If the values - do not fit in decimal, then it infers them as doubles. If None is - set, it uses the default value, ``false``. - allowComments : str or bool, optional - ignores Java/C++ style comment in JSON records. If None is set, - it uses the default value, ``false``. - allowUnquotedFieldNames : str or bool, optional - allows unquoted JSON field names. If None is set, - it uses the default value, ``false``. - allowSingleQuotes : str or bool, optional - allows single quotes in addition to double quotes. If None is - set, it uses the default value, ``true``. - allowNumericLeadingZero : str or bool, optional - allows leading zeros in numbers (e.g. 00012). If None is - set, it uses the default value, ``false``. - allowBackslashEscapingAnyCharacter : str or bool, optional - allows accepting quoting of all character - using backslash quoting mechanism. If None is - set, it uses the default value, ``false``. - mode : str, optional - allows a mode for dealing with corrupt records during parsing. If None is - set, it uses the default value, ``PERMISSIVE``. - - * ``PERMISSIVE``: when it meets a corrupted record, puts the malformed string \ - into a field configured by ``columnNameOfCorruptRecord``, and sets malformed \ - fields to ``null``. To keep corrupt records, an user can set a string type \ - field named ``columnNameOfCorruptRecord`` in an user-defined schema. If a \ - schema does not have the field, it drops corrupt records during parsing. \ - When inferring a schema, it implicitly adds a ``columnNameOfCorruptRecord`` \ - field in an output schema. - * ``DROPMALFORMED``: ignores the whole corrupted records. - * ``FAILFAST``: throws an exception when it meets corrupted records. - - columnNameOfCorruptRecord : str, optional - allows renaming the new field having malformed string - created by ``PERMISSIVE`` mode. This overrides - ``spark.sql.columnNameOfCorruptRecord``. If None is set, - it uses the value specified in - ``spark.sql.columnNameOfCorruptRecord``. - dateFormat : str, optional - sets the string that indicates a date format. Custom date formats - follow the formats at - `datetime pattern `_. # noqa - This applies to date type. If None is set, it uses the - default value, ``yyyy-MM-dd``. - timestampFormat : str, optional - sets the string that indicates a timestamp format. - Custom date formats follow the formats at - `datetime pattern `_. # noqa - This applies to timestamp type. If None is set, it uses the - default value, ``yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]``. - multiLine : str or bool, optional - parse one record, which may span multiple lines, per file. If None is - set, it uses the default value, ``false``. - allowUnquotedControlChars : str or bool, optional - allows JSON Strings to contain unquoted control - characters (ASCII characters with value less than 32, - including tab and line feed characters) or not. - lineSep : str, optional - defines the line separator that should be used for parsing. If None is - set, it covers all ``\\r``, ``\\r\\n`` and ``\\n``. - locale : str, optional - sets a locale as language tag in IETF BCP 47 format. If None is set, - it uses the default value, ``en-US``. For instance, ``locale`` is used while - parsing dates and timestamps. - dropFieldIfAllNull : str or bool, optional - whether to ignore column of all null values or empty - array/struct during schema inference. If None is set, it - uses the default value, ``false``. - encoding : str or bool, optional - allows to forcibly set one of standard basic or extended encoding for - the JSON files. For example UTF-16BE, UTF-32LE. If None is set, - the encoding of input JSON will be detected automatically - when the multiLine option is set to ``true``. - pathGlobFilter : str or bool, optional - an optional glob pattern to only include files with paths matching - the pattern. The syntax follows `org.apache.hadoop.fs.GlobFilter`. - It does not change the behavior of - `partition discovery `_. # noqa - recursiveFileLookup : str or bool, optional - recursively scan a directory for files. Using this option - disables - `partition discovery `_. # noqa - allowNonNumericNumbers : str or bool, optional - allows JSON parser to recognize set of "Not-a-Number" (NaN) - tokens as legal floating number values. If None is set, - it uses the default value, ``true``. - * ``+INF``: for positive infinity, as well as alias of - ``+Infinity`` and ``Infinity``. - * ``-INF``: for negative infinity, alias ``-Infinity``. - * ``NaN``: for other not-a-numbers, like result of division by zero. + Other Parameters + ---------------- + Extra options + For the extra options, refer to + `Data Source Option `_ # noqa + in the version you use. Notes ----- @@ -1083,20 +966,6 @@ def format(self, source): def option(self, key, value): """Adds an output option for the underlying data source. - You can set the following option(s) for writing files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to format - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - .. versionadded:: 2.0.0 Notes @@ -1109,20 +978,6 @@ def option(self, key, value): def options(self, **options): """Adds output options for the underlying data source. - You can set the following option(s) for writing files: - * ``timeZone``: sets the string that indicates a time zone ID to be used to format - timestamps in the JSON/CSV datasources or partition values. The following - formats of `timeZone` are supported: - - * Region-based zone ID: It should have the form 'area/city', such as \ - 'America/Los_Angeles'. - * Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' or \ - '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'. - - Other short names like 'CST' are not recommended to use because they can be - ambiguous. If it isn't set, the current value of the SQL config - ``spark.sql.session.timeZone`` is used by default. - .. versionadded:: 2.0.0 Notes diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala index f7e1903da687..bb6a3e4f3ecb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala @@ -101,23 +101,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def option(key: String, value: String): DataFrameReader = { @@ -161,23 +144,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def options(options: scala.collection.Map[String, String]): DataFrameReader = { @@ -191,23 +157,6 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def options(options: java.util.Map[String, String]): DataFrameReader = { @@ -441,81 +390,9 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging { * This function goes through the input once to determine the input schema. If you know the * schema in advance, use the version that specifies the schema to avoid the extra scan. * - * You can set the following JSON-specific options to deal with non-standard JSON files: - *
    - *
  • `primitivesAsString` (default `false`): infers all primitive values as a string type
    • - *
  • `prefersDecimal` (default `false`): infers all floating-point values as a decimal - * type. If the values do not fit in decimal, then it infers them as doubles.
    • - *
  • `allowComments` (default `false`): ignores Java/C++ style comment in JSON records
    • - *
  • `allowUnquotedFieldNames` (default `false`): allows unquoted JSON field names
    • - *
  • `allowSingleQuotes` (default `true`): allows single quotes in addition to double quotes - *
    • - *
  • `allowNumericLeadingZeros` (default `false`): allows leading zeros in numbers - * (e.g. 00012)
    • - *
  • `allowBackslashEscapingAnyCharacter` (default `false`): allows accepting quoting of all - * character using backslash quoting mechanism
    • - *
  • `allowUnquotedControlChars` (default `false`): allows JSON Strings to contain unquoted - * control characters (ASCII characters with value less than 32, including tab and line feed - * characters) or not.
    • - *
  • `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records - * during parsing. - *
      - *
    • `PERMISSIVE` : when it meets a corrupted record, puts the malformed string into a - * field configured by `columnNameOfCorruptRecord`, and sets malformed fields to `null`. To - * keep corrupt records, an user can set a string type field named - * `columnNameOfCorruptRecord` in an user-defined schema. If a schema does not have the - * field, it drops corrupt records during parsing. When inferring a schema, it implicitly - * adds a `columnNameOfCorruptRecord` field in an output schema.
      • - *
    • `DROPMALFORMED` : ignores the whole corrupted records.
      • - *
    • `FAILFAST` : throws an exception when it meets corrupted records.
      • - *
    - *
    • - *
  • `columnNameOfCorruptRecord` (default is the value specified in - * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string - * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.
    • - *
  • `dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format. - * Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to date type.
    • - *
  • `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that - * indicates a timestamp format. Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to timestamp type.
    • - *
  • `multiLine` (default `false`): parse one record, which may span multiple lines, - * per file
    • - *
  • `encoding` (by default it is not set): allows to forcibly set one of standard basic - * or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE. If the encoding - * is not specified and `multiLine` is set to `true`, it will be detected automatically.
    • - *
  • `lineSep` (default covers all `\r`, `\r\n` and `\n`): defines the line separator - * that should be used for parsing.
    • - *
  • `samplingRatio` (default is 1.0): defines fraction of input JSON objects used - * for schema inferring.
    • - *
  • `dropFieldIfAllNull` (default `false`): whether to ignore column of all null values or - * empty array/struct during schema inference.
    • - *
  • `locale` (default is `en-US`): sets a locale as language tag in IETF BCP 47 format. - * For instance, this is used while parsing dates and timestamps.
    • - *
  • `pathGlobFilter`: an optional glob pattern to only include files with paths matching - * the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter. - * It does not change the behavior of partition discovery.
    • - *
  • `modifiedBefore` (batch only): an optional timestamp to only include files with - * modification times occurring before the specified Time. The provided timestamp - * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
    • - *
  • `modifiedAfter` (batch only): an optional timestamp to only include files with - * modification times occurring after the specified Time. The provided timestamp - * must be in the following form: YYYY-MM-DDTHH:mm:ss (e.g. 2020-06-01T13:00:00)
    • - *
  • `recursiveFileLookup`: recursively scan a directory for files. Using this option - * disables partition discovery
    • - *
  • `allowNonNumericNumbers` (default `true`): allows JSON parser to recognize set of - * "Not-a-Number" (NaN) tokens as legal floating number values: - *
      - *
    • `+INF` for positive infinity, as well as alias of `+Infinity` and `Infinity`. - *
    • `-INF` for negative infinity), alias `-Infinity`. - *
    • `NaN` for other not-a-numbers, like result of division by zero. - *
    - *
    • - *
+ * You can find the JSON-specific options for reading JSON files in + * + * Data Source Option in the version you use. * * @since 2.0.0 */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala index fe6572cff5de..09dc60f7c42e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala @@ -108,23 +108,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def option(key: String, value: String): DataFrameWriter[T] = { @@ -168,23 +151,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def options(options: scala.collection.Map[String, String]): DataFrameWriter[T] = { @@ -198,23 +164,6 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { * All options are maintained in a case-insensitive way in terms of key names. * If a new option has the same key case-insensitively, it will override the existing option. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 1.4.0 */ def options(options: java.util.Map[String, String]): DataFrameWriter[T] = { @@ -825,27 +774,9 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) { * format("json").save(path) * }}} * - * You can set the following JSON-specific option(s) for writing JSON files: - *
    - *
  • `compression` (default `null`): compression codec to use when saving to file. This can be - * one of the known case-insensitive shorten names (`none`, `bzip2`, `gzip`, `lz4`, - * `snappy` and `deflate`).
    • - *
  • `dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format. - * Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to date type.
    • - *
  • `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that - * indicates a timestamp format. Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to timestamp type.
    • - *
  • `encoding` (by default it is not set): specifies encoding (charset) of saved json - * files. If it is not set, the UTF-8 charset will be used.
    • - *
  • `lineSep` (default `\n`): defines the line separator that should be used for writing.
    • - *
  • `ignoreNullFields` (default `true`): Whether to ignore null fields - * when generating JSON objects.
    • - *
+ * You can find the JSON-specific options for writing JSON files in + * + * Data Source Option in the version you use. * * @since 1.4.0 */ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index bd222d150d46..1e763a212dbd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -4123,6 +4123,7 @@ object functions { JsonTuple(json.expr +: fields.map(Literal.apply)) } + // scalastyle:off line.size.limit /** * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the * specified schema. Returns `null`, in the case of an unparseable string. @@ -4131,13 +4132,19 @@ object functions { * @param schema the schema to use when parsing the json string * @param options options to control how the json is parsed. Accepts the same options as the * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.1.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = from_json(e, schema.asInstanceOf[DataType], options) + // scalastyle:off line.size.limit /** * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` * as keys type, `StructType` or `ArrayType` with the specified schema. @@ -4147,14 +4154,20 @@ object functions { * @param schema the schema to use when parsing the json string * @param options options to control how the json is parsed. accepts the same options and the * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.2.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = withExpr { JsonToStructs(CharVarcharUtils.failIfHasCharVarchar(schema), options, e.expr) } + // scalastyle:off line.size.limit /** * (Java-specific) Parses a column containing a JSON string into a `StructType` with the * specified schema. Returns `null`, in the case of an unparseable string. @@ -4163,13 +4176,19 @@ object functions { * @param schema the schema to use when parsing the json string * @param options options to control how the json is parsed. accepts the same options and the * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.1.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = from_json(e, schema, options.asScala.toMap) + // scalastyle:off line.size.limit /** * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` * as keys type, `StructType` or `ArrayType` with the specified schema. @@ -4179,10 +4198,15 @@ object functions { * @param schema the schema to use when parsing the json string * @param options options to control how the json is parsed. accepts the same options and the * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.2.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { from_json(e, CharVarcharUtils.failIfHasCharVarchar(schema), options.asScala.toMap) } @@ -4214,6 +4238,7 @@ object functions { def from_json(e: Column, schema: DataType): Column = from_json(e, schema, Map.empty[String, String]) + // scalastyle:off line.size.limit /** * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` * as keys type, `StructType` or `ArrayType` with the specified schema. @@ -4221,14 +4246,22 @@ object functions { * * @param e a string column containing JSON data. * @param schema the schema as a DDL-formatted string. + * @param options options to control how the json is parsed. accepts the same options and the + * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.1.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { from_json(e, schema, options.asScala.toMap) } + // scalastyle:off line.size.limit /** * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` * as keys type, `StructType` or `ArrayType` with the specified schema. @@ -4236,10 +4269,17 @@ object functions { * * @param e a string column containing JSON data. * @param schema the schema as a DDL-formatted string. + * @param options options to control how the json is parsed. accepts the same options and the + * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.3.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: Map[String, String]): Column = { val dataType = parseTypeWithFallback( schema, @@ -4264,6 +4304,7 @@ object functions { from_json(e, schema, Map.empty[String, String].asJava) } + // scalastyle:off line.size.limit /** * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. @@ -4273,10 +4314,15 @@ object functions { * @param schema the schema to use when parsing the json string * @param options options to control how the json is parsed. accepts the same options and the * json data source. + * See + * + * Data Source Option in the version you use. * * @group collection_funcs * @since 2.4.0 */ + // scalastyle:on line.size.limit def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { withExpr(new JsonToStructs(e.expr, schema.expr, options.asScala.toMap)) } @@ -4301,21 +4347,28 @@ object functions { */ def schema_of_json(json: Column): Column = withExpr(new SchemaOfJson(json.expr)) + // scalastyle:off line.size.limit /** * Parses a JSON string and infers its schema in DDL format using options. * * @param json a foldable string column containing JSON data. * @param options options to control how the json is parsed. accepts the same options and the - * json data source. See [[DataFrameReader#json]]. + * json data source. + * See + * + * Data Source Option in the version you use. * @return a column with string literal containing schema in DDL format. * * @group collection_funcs * @since 3.0.0 */ + // scalastyle:on line.size.limit def schema_of_json(json: Column, options: java.util.Map[String, String]): Column = { withExpr(SchemaOfJson(json.expr, options.asScala.toMap)) } + // scalastyle:off line.size.limit /** * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or * a `MapType` into a JSON string with the specified schema. @@ -4324,16 +4377,22 @@ object functions { * @param e a column containing a struct, an array or a map. * @param options options to control how the struct column is converted into a json string. * accepts the same options and the json data source. + * See + * + * Data Source Option in the version you use. * Additionally the function supports the `pretty` option which enables * pretty JSON generation. * * @group collection_funcs * @since 2.1.0 */ + // scalastyle:on line.size.limit def to_json(e: Column, options: Map[String, String]): Column = withExpr { StructsToJson(options, e.expr) } + // scalastyle:off line.size.limit /** * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or * a `MapType` into a JSON string with the specified schema. @@ -4342,12 +4401,17 @@ object functions { * @param e a column containing a struct, an array or a map. * @param options options to control how the struct column is converted into a json string. * accepts the same options and the json data source. + * See + * + * Data Source Option in the version you use. * Additionally the function supports the `pretty` option which enables * pretty JSON generation. * * @group collection_funcs * @since 2.1.0 */ + // scalastyle:on line.size.limit def to_json(e: Column, options: java.util.Map[String, String]): Column = to_json(e, options.asScala.toMap) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala index 1798f6e2c88b..9c71101e4d9d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamReader.scala @@ -85,23 +85,6 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo /** * Adds an input option for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def option(key: String, value: String): DataStreamReader = { @@ -133,23 +116,6 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo /** * (Scala-specific) Adds input options for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def options(options: scala.collection.Map[String, String]): DataStreamReader = { @@ -160,23 +126,6 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo /** * (Java-specific) Adds input options for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to parse timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def options(options: java.util.Map[String, String]): DataStreamReader = { @@ -269,73 +218,16 @@ final class DataStreamReader private[sql](sparkSession: SparkSession) extends Lo * This function goes through the input once to determine the input schema. If you know the * schema in advance, use the version that specifies the schema to avoid the extra scan. * - * You can set the following JSON-specific options to deal with non-standard JSON files: + * You can set the following option(s): *
    *
  • `maxFilesPerTrigger` (default: no max limit): sets the maximum number of new files to be * considered in every trigger.
    • - *
  • `primitivesAsString` (default `false`): infers all primitive values as a string type
    • - *
  • `prefersDecimal` (default `false`): infers all floating-point values as a decimal - * type. If the values do not fit in decimal, then it infers them as doubles.
    • - *
  • `allowComments` (default `false`): ignores Java/C++ style comment in JSON records
    • - *
  • `allowUnquotedFieldNames` (default `false`): allows unquoted JSON field names
    • - *
  • `allowSingleQuotes` (default `true`): allows single quotes in addition to double quotes - *
    • - *
  • `allowNumericLeadingZeros` (default `false`): allows leading zeros in numbers - * (e.g. 00012)
    • - *
  • `allowBackslashEscapingAnyCharacter` (default `false`): allows accepting quoting of all - * character using backslash quoting mechanism
    • - *
  • `allowUnquotedControlChars` (default `false`): allows JSON Strings to contain unquoted - * control characters (ASCII characters with value less than 32, including tab and line feed - * characters) or not.
    • - *
  • `mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records - * during parsing. - *
      - *
    • `PERMISSIVE` : when it meets a corrupted record, puts the malformed string into a - * field configured by `columnNameOfCorruptRecord`, and sets malformed fields to `null`. To - * keep corrupt records, an user can set a string type field named - * `columnNameOfCorruptRecord` in an user-defined schema. If a schema does not have the - * field, it drops corrupt records during parsing. When inferring a schema, it implicitly - * adds a `columnNameOfCorruptRecord` field in an output schema.
      • - *
    • `DROPMALFORMED` : ignores the whole corrupted records.
      • - *
    • `FAILFAST` : throws an exception when it meets corrupted records.
      • - *
    - *
    • - *
  • `columnNameOfCorruptRecord` (default is the value specified in - * `spark.sql.columnNameOfCorruptRecord`): allows renaming the new field having malformed string - * created by `PERMISSIVE` mode. This overrides `spark.sql.columnNameOfCorruptRecord`.
    • - *
  • `dateFormat` (default `yyyy-MM-dd`): sets the string that indicates a date format. - * Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to date type.
    • - *
  • `timestampFormat` (default `yyyy-MM-dd'T'HH:mm:ss[.SSS][XXX]`): sets the string that - * indicates a timestamp format. Custom date formats follow the formats at - * - * Datetime Patterns. - * This applies to timestamp type.
    • - *
  • `multiLine` (default `false`): parse one record, which may span multiple lines, - * per file
    • - *
  • `lineSep` (default covers all `\r`, `\r\n` and `\n`): defines the line separator - * that should be used for parsing.
    • - *
  • `dropFieldIfAllNull` (default `false`): whether to ignore column of all null values or - * empty array/struct during schema inference.
    • - *
  • `locale` (default is `en-US`): sets a locale as language tag in IETF BCP 47 format. - * For instance, this is used while parsing dates and timestamps.
    • - *
  • `pathGlobFilter`: an optional glob pattern to only include files with paths matching - * the pattern. The syntax follows org.apache.hadoop.fs.GlobFilter. - * It does not change the behavior of partition discovery.
    • - *
  • `recursiveFileLookup`: recursively scan a directory for files. Using this option - * disables partition discovery
    • - *
  • `allowNonNumericNumbers` (default `true`): allows JSON parser to recognize set of - * "Not-a-Number" (NaN) tokens as legal floating number values: - *
      - *
    • `+INF` for positive infinity, as well as alias of `+Infinity` and `Infinity`. - *
    • `-INF` for negative infinity, alias `-Infinity`. - *
    • `NaN` for other not-a-numbers, like result of division by zero. - *
    - *
    • *
* + * You can find the JSON-specific options for reading JSON file stream in + * + * Data Source Option in the version you use. + * * @since 2.0.0 */ def json(path: String): DataFrame = format("json").load(path) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala index b25aedbeca79..10ce9d3aaf01 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/streaming/DataStreamWriter.scala @@ -167,23 +167,6 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) { /** * Adds an output option for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def option(key: String, value: String): DataStreamWriter[T] = { @@ -215,23 +198,6 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) { /** * (Scala-specific) Adds output options for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def options(options: scala.collection.Map[String, String]): DataStreamWriter[T] = { @@ -242,23 +208,6 @@ final class DataStreamWriter[T] private[sql](ds: Dataset[T]) { /** * Adds output options for the underlying data source. * - * You can set the following option(s): - *
    - *
  • `timeZone` (default session local timezone): sets the string that indicates a time zone ID - * to be used to format timestamps in the JSON/CSV datasources or partition values. The following - * formats of `timeZone` are supported: - *
      - *
    • Region-based zone ID: It should have the form 'area/city', such as - * 'America/Los_Angeles'.
      • - *
    • Zone offset: It should be in the format '(+|-)HH:mm', for example '-08:00' - * or '+01:00'. Also 'UTC' and 'Z' are supported as aliases of '+00:00'.
      • - *
    - * Other short names like 'CST' are not recommended to use because they can be ambiguous. - * If it isn't set, the current value of the SQL config `spark.sql.session.timeZone` is - * used by default. - *
    • - *
- * * @since 2.0.0 */ def options(options: java.util.Map[String, String]): DataStreamWriter[T] = {