From 4a6f903897d28a3038918997e692410259a90ae3 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Fri, 19 Jun 2020 10:36:52 +0800 Subject: [PATCH 1/8] Reuse completeNextStageWithFetchFailure --- .../apache/spark/scheduler/DAGSchedulerSuite.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index 9d412f2dba3ce..762b14e170fcc 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -1796,9 +1796,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi // lets say there is a fetch failure in this task set, which makes us go back and // run stage 0, attempt 1 - complete(taskSets(1), Seq( - (FetchFailed(makeBlockManagerId("hostA"), - shuffleDep1.shuffleId, 0L, 0, 0, "ignored"), null))) + completeNextStageWithFetchFailure(1, 0, shuffleDep1) scheduler.resubmitFailedStages() // stage 0, attempt 1 should have the properties of job2 @@ -1872,9 +1870,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi // have the second stage complete normally completeShuffleMapStageSuccessfully(1, 0, 1, Seq("hostA", "hostC")) // fail the third stage because hostA went down - complete(taskSets(2), Seq( - (FetchFailed(makeBlockManagerId("hostA"), - shuffleDepTwo.shuffleId, 0L, 0, 0, "ignored"), null))) + completeNextStageWithFetchFailure(2, 0, shuffleDepTwo) // TODO assert this: // blockManagerMaster.removeExecutor("hostA-exec") // have DAGScheduler try again @@ -1900,9 +1896,7 @@ class DAGSchedulerSuite extends SparkFunSuite with LocalSparkContext with TimeLi // complete stage 1 completeShuffleMapStageSuccessfully(1, 0, 1) // pretend stage 2 failed because hostA went down - complete(taskSets(2), Seq( - (FetchFailed(makeBlockManagerId("hostA"), - shuffleDepTwo.shuffleId, 0L, 0, 0, "ignored"), null))) + completeNextStageWithFetchFailure(2, 0, shuffleDepTwo) // TODO assert this: // blockManagerMaster.removeExecutor("hostA-exec") // DAGScheduler should notice the cached copy of the second shuffle and try to get it rerun. From 1345e495135b2047e32480d2a63326edd9f0c9fc Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 4 Jan 2021 18:16:58 +0800 Subject: [PATCH 2/8] Add doc for like any|all operators --- docs/sql-ref-syntax-qry-select-like.md | 60 +++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index 6211faa8d529e..88a6ecde3fa75 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,12 +21,14 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. +A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers includes `SOME`, `ANY`, `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true. ### Syntax ```sql [ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | [ RLIKE | REGEXP ] regex_pattern } + +[ NOT ] { LIKE predicate_quantifiers ( search_pattern [ , ... ]) } ``` ### Parameters @@ -45,6 +47,10 @@ A LIKE predicate is used to search for a specific pattern. * **regex_pattern** Specifies a regular expression search pattern to be searched by the `RLIKE` or `REGEXP` clause. + +* **predicate_quantifiers** + + Specifies the predicate quantifiers include `SOME`, `ANY`, `ALL`. ### Examples @@ -111,6 +117,58 @@ SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$'; +---+------+---+ |500|Evan_W| 16| +---+------+---+ + +SELECT * FROM person WHERE name LIKE ALL ('%an%', '%an'); ++---+----+----+ +| id|name| age| ++---+----+----+ +|400| Dan| 50| ++---+----+----+ + +SELECT * FROM person WHERE name LIKE ANY ('%an%', '%an'); ++---+------+---+ +| id| name|age| ++---+------+---+ +|400| Dan| 50| +|500|Evan_W| 16| ++---+------+---+ + +SELECT * FROM person WHERE name LIKE SOME ('%an%', '%an'); ++---+------+---+ +| id| name|age| ++---+------+---+ +|400| Dan| 50| +|500|Evan_W| 16| ++---+------+---+ + +SELECT * FROM person WHERE name NOT LIKE ALL ('%an%', '%an'); ++---+----+----+ +| id|name| age| ++---+----+----+ +|100|John| 30| +|200|Mary|null| +|300|Mike| 80| ++---+----+----+ + +SELECT * FROM person WHERE name NOT LIKE ANY ('%an%', '%an'); ++---+------+---+ +| id| name| age| ++---+------+----+ +|100| John| 30| +|200| Mary|null| +|300| Mike| 80| +|500|Evan_W| 16| ++---+------+----+ + +SELECT * FROM person WHERE name NOT LIKE SOME ('%an%', '%an'); ++---+------+---+ +| id| name| age| ++---+------+----+ +|100| John| 30| +|200| Mary|null| +|300| Mike| 80| +|500|Evan_W| 16| ++---+------+----+ ``` ### Related Statements From e159efc9b6db0b8eb0ef3cdabcf85110ce72d491 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 4 Jan 2021 18:21:46 +0800 Subject: [PATCH 3/8] Add doc for like any|all operators --- docs/sql-ref-syntax-qry-select-like.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index 88a6ecde3fa75..d5a211e521924 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,7 +21,7 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers includes `SOME`, `ANY`, `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true. +A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers includes `SOME`, `ANY`, `ALL`. ### Syntax @@ -50,7 +50,7 @@ A LIKE predicate is used to search for a specific pattern. LIKE predicate also s * **predicate_quantifiers** - Specifies the predicate quantifiers include `SOME`, `ANY`, `ALL`. + Specifies the predicate quantifiers include `SOME`, `ANY`, `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true. ### Examples From 3d0826e6c522648dd75ac2f44aa6b5dc8766f647 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Mon, 4 Jan 2021 18:24:33 +0800 Subject: [PATCH 4/8] Add doc for like any|all operators --- docs/sql-ref-syntax-qry-select-like.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index d5a211e521924..f4977dc8317b8 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,7 +21,7 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers includes `SOME`, `ANY`, `ALL`. +A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers include `SOME`, `ANY`, `ALL`. ### Syntax From 348c564223e02e7cba092f3d3e83bf0c8f609c77 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Tue, 5 Jan 2021 10:18:09 +0800 Subject: [PATCH 5/8] Update doc --- docs/sql-ref-syntax-qry-select-like.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index f4977dc8317b8..a7cb005902810 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,14 +21,14 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers include `SOME`, `ANY`, `ALL`. +A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers include `ANY`, `SOME` and `ALL`. ### Syntax ```sql [ NOT ] { LIKE search_pattern [ ESCAPE esc_char ] | [ RLIKE | REGEXP ] regex_pattern } -[ NOT ] { LIKE predicate_quantifiers ( search_pattern [ , ... ]) } +[ NOT ] { LIKE quantifiers ( search_pattern [ , ... ]) } ``` ### Parameters @@ -48,9 +48,9 @@ A LIKE predicate is used to search for a specific pattern. LIKE predicate also s Specifies a regular expression search pattern to be searched by the `RLIKE` or `REGEXP` clause. -* **predicate_quantifiers** +* **quantifiers** - Specifies the predicate quantifiers include `SOME`, `ANY`, `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true. + Specifies the predicate quantifiers include `ANY`, `SOME` and `ALL`. `ANY` or `SOME` means if one of the patterns matches the input, then return true; `ALL` means if all the patterns matches the input, then return true. ### Examples From a52e46f11e550628b7bf6f583247dc83228a299a Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Tue, 5 Jan 2021 10:19:51 +0800 Subject: [PATCH 6/8] Update doc --- docs/sql-ref-syntax-qry-select-like.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index a7cb005902810..5131788ca8f6b 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -151,7 +151,7 @@ SELECT * FROM person WHERE name NOT LIKE ALL ('%an%', '%an'); +---+----+----+ SELECT * FROM person WHERE name NOT LIKE ANY ('%an%', '%an'); -+---+------+---+ ++---+------+----+ | id| name| age| +---+------+----+ |100| John| 30| @@ -161,7 +161,7 @@ SELECT * FROM person WHERE name NOT LIKE ANY ('%an%', '%an'); +---+------+----+ SELECT * FROM person WHERE name NOT LIKE SOME ('%an%', '%an'); -+---+------+---+ ++---+------+----+ | id| name| age| +---+------+----+ |100| John| 30| From d5b0e482b2d4abd89997039cccce81458408d14c Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Tue, 5 Jan 2021 10:21:26 +0800 Subject: [PATCH 7/8] Update doc --- docs/sql-ref-syntax-qry-select-like.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index 5131788ca8f6b..ca2c3239f7ed6 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,7 +21,7 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with predicate quantifiers include `ANY`, `SOME` and `ALL`. +A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with quantifiers include `ANY`, `SOME` and `ALL`. ### Syntax From 524759aea95d2de99773da6eb76366b0d22725c5 Mon Sep 17 00:00:00 2001 From: gengjiaan Date: Wed, 6 Jan 2021 10:26:13 +0800 Subject: [PATCH 8/8] Update doc --- docs/sql-ref-syntax-qry-select-like.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-ref-syntax-qry-select-like.md b/docs/sql-ref-syntax-qry-select-like.md index ca2c3239f7ed6..3604a9ba1ea02 100644 --- a/docs/sql-ref-syntax-qry-select-like.md +++ b/docs/sql-ref-syntax-qry-select-like.md @@ -21,7 +21,7 @@ license: | ### Description -A LIKE predicate is used to search for a specific pattern. LIKE predicate also supports multiple patterns with quantifiers include `ANY`, `SOME` and `ALL`. +A LIKE predicate is used to search for a specific pattern. This predicate also supports multiple patterns with quantifiers include `ANY`, `SOME` and `ALL`. ### Syntax