Skip to content

Commit 0f489dd

Browse files
CalvinKirsCalvinKirs
authored
[test](kerberos)Add hdfs(tvf),outfile and export test (#48433) (#48776)
#48433 (cherry picked from commit 48fa090)
1 parent 48e3a73 commit 0f489dd

File tree

3 files changed

+154
-0
lines changed

3 files changed

+154
-0
lines changed

‎regression-test/data/external_table_p0/kerberos/test_non_catalog_kerberos.out

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !select1 --
3+
1 2
4+
5+
-- !select1 --
6+
1 2
7+

‎regression-test/pipeline/external/conf/regression-conf.groovy

+1
Original file line numberDiff line numberDiff line change
@@ -140,3 +140,4 @@ externalEnvIp="127.0.0.1"
140140
enableKerberosTest = true
141141
kerberosHmsPort=9883
142142
kerberosHdfsPort=8820
143+
enableNonCatalogKerberosTest = true

‎regression-test/suites/external_table_p0/kerberos/test_non_catalog_kerberos.groovy

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.awaitility.Awaitility;
19+
import static java.util.concurrent.TimeUnit.SECONDS;
20+
21+
suite("test_non_catalog_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") {
22+
String enabled = context.config.otherConfigs.get("enableNonCatalogKerberosTest")
23+
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
24+
return
25+
}
26+
def String hms_catalog_name = "hms_catalog_kerberos_test_export"
27+
def String test_tbl_name="hms_test_table"
28+
def keytab_root_dir = "/keytabs"
29+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
30+
sql """
31+
drop catalog if exists ${hms_catalog_name}
32+
"""
33+
sql """
34+
CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
35+
PROPERTIES (
36+
"type" = "hms",
37+
"ipc.client.fallback-to-simple-auth-allowed" = "true",
38+
"hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
39+
"hive.metastore.sasl.enabled " = "true",
40+
"hive.metastore.kerberos.principal" = "hive/[email protected]",
41+
"hadoop.security.authentication" = "kerberos",
42+
"hadoop.security.auth_to_local" = "RULE:[2:\$1@\$0](.*@LABS.TERADATA.COM)s/@.*//
43+
RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
44+
RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*//
45+
DEFAULT",
46+
"hadoop.kerberos.principal" = "hive/[email protected]",
47+
"hadoop.kerberos.min.seconds.before.relogin" = "5",
48+
"hadoop.kerberos.keytab.login.autorenewal.enabled" = "false",
49+
"hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab",
50+
"fs.defaultFS" = "hdfs://${externalEnvIp}:8520"
51+
);
52+
"""
53+
54+
sql """ switch ${hms_catalog_name} """
55+
sql """ use test_krb_hive_db """
56+
sql """ drop table if exists ${test_tbl_name}"""
57+
sql """
58+
CREATE TABLE `${test_tbl_name}` (
59+
`c_tinyint` tinyint(4) NULL COMMENT "",
60+
`c_smallint` smallint(6) NULL COMMENT ""
61+
) ;
62+
"""
63+
sql """
64+
insert into ${test_tbl_name} values(1,2);
65+
"""
66+
qt_select1 "select * from ${test_tbl_name} "
67+
def export_task_label= "export_kerberos_test"+ System.currentTimeMillis()
68+
69+
sql """
70+
EXPORT TABLE ${test_tbl_name}
71+
TO "hdfs://${externalEnvIp}:8520/user/test/export_"
72+
PROPERTIES
73+
(
74+
"line_delimiter" = ",",
75+
"label"="${export_task_label}"
76+
)
77+
with HDFS (
78+
"fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
79+
"hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
80+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
81+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
82+
DEFAULT",
83+
"hadoop.kerberos.min.seconds.before.relogin" = "5",
84+
"hadoop.security.authentication" = "kerberos",
85+
"hadoop.kerberos.keytab.login.autorenewal.enabled"="false",
86+
"hadoop.kerberos.principal"="hive/[email protected]",
87+
"hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab"
88+
);
89+
"""
90+
91+
def outfile_result=sql """
92+
SELECT * FROM ${test_tbl_name}
93+
INTO OUTFILE "hdfs://${externalEnvIp}:8520/user/to/outfile_"
94+
FORMAT AS CSV
95+
PROPERTIES(
96+
"fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
97+
"hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
98+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
99+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
100+
DEFAULT",
101+
"hadoop.kerberos.min.seconds.before.relogin" = "5",
102+
"hadoop.security.authentication" = "kerberos",
103+
"hadoop.kerberos.keytab.login.autorenewal.enabled"="false",
104+
"hadoop.kerberos.principal"="hive/[email protected]",
105+
"hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab"
106+
)
107+
"""
108+
109+
println(outfile_result)
110+
def hdfslink=outfile_result.get(0).get(3)
111+
println hdfslink
112+
qt_select1 """
113+
select * from hdfs(
114+
"uri" = "${hdfslink}",
115+
"hadoop.username" = "doris",
116+
"format" = "csv",
117+
"fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
118+
"hadoop.security.auth_to_local" = "RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
119+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
120+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
121+
DEFAULT",
122+
"hadoop.kerberos.min.seconds.before.relogin" = "5",
123+
"hadoop.security.authentication" = "kerberos",
124+
"hadoop.kerberos.keytab.login.autorenewal.enabled"="false",
125+
"hadoop.kerberos.principal"="hive/[email protected]",
126+
"hadoop.kerberos.keytab" = "${keytab_root_dir}/hive-presto-master.keytab"
127+
)
128+
129+
"""
130+
Awaitility.await("queery-export-task-result-test").atMost(60, SECONDS).pollInterval(5, SECONDS).until(
131+
{
132+
sql """ switch ${hms_catalog_name} """
133+
sql """ use test_krb_hive_db """
134+
def res = sql """ show export where label = "${export_task_label}" """
135+
if (res[0][2] == "FINISHED") {
136+
return true
137+
} else if (res[0][2] == "CANCELLED") {
138+
throw new IllegalStateException("""export failed: ${res[0][10]}""")
139+
} else {
140+
return false
141+
}
142+
}
143+
)
144+
145+
146+
}

0 commit comments

Comments
 (0)