-
Notifications
You must be signed in to change notification settings - Fork 4.8k
HIVE-29178: Add Catalog properties support #6123
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
c8848d9
11dd799
34957a4
d955426
ff309a8
dc1491e
8203c56
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -159,6 +159,7 @@ TOK_UNIONTYPE; | |
TOK_VARIANT; | ||
TOK_COLTYPELIST; | ||
TOK_CREATECATALOG; | ||
TOK_CATALOGPROPERTIES; | ||
TOK_CREATEDATABASE; | ||
TOK_CREATEDATACONNECTOR; | ||
TOK_CREATETABLE; | ||
|
@@ -378,6 +379,7 @@ TOK_DESCCATALOG; | |
TOK_CATALOGLOCATION; | ||
TOK_CATALOGCOMMENT; | ||
TOK_ALTERCATALOG_LOCATION; | ||
TOK_ALTERCATALOG_PROPERTIES; | ||
TOK_DESCDATABASE; | ||
TOK_DATABASEPROPERTIES; | ||
TOK_DATABASELOCATION; | ||
|
@@ -682,7 +684,7 @@ import org.apache.hadoop.hive.conf.HiveConf; | |
xlateMap.put("KW_LIMIT", "LIMIT"); | ||
xlateMap.put("KW_OFFSET", "OFFSET"); | ||
xlateMap.put("KW_SET", "SET"); | ||
xlateMap.put("KW_PROPERTIES", "TBLPROPERTIES"); | ||
xlateMap.put("KW_PROPERTIES", "PROPERTIES"); | ||
xlateMap.put("KW_VALUE_TYPE", "\$VALUE\$"); | ||
xlateMap.put("KW_ELEM_TYPE", "\$ELEM\$"); | ||
xlateMap.put("KW_DEFINED", "DEFINED"); | ||
|
@@ -1127,7 +1129,8 @@ createCatalogStatement | |
name=identifier | ||
catLocation | ||
catalogComment? | ||
-> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment?) | ||
(KW_PROPERTIES catprops=catProperties)? | ||
-> ^(TOK_CREATECATALOG $name catLocation ifNotExists? catalogComment? $catprops?) | ||
; | ||
|
||
catLocation | ||
|
@@ -1144,6 +1147,13 @@ catalogComment | |
-> ^(TOK_CATALOGCOMMENT $comment) | ||
; | ||
|
||
catProperties | ||
@init { pushMsg("catproperties", state); } | ||
@after { popMsg(state); } | ||
: | ||
LPAREN dbPropertiesList RPAREN -> ^(TOK_CATALOGPROPERTIES dbPropertiesList) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe |
||
; | ||
|
||
dropCatalogStatement | ||
@init { pushMsg("drop catalog statement", state); } | ||
@after { popMsg(state); } | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -974,7 +974,7 @@ nonReserved | |
KW_ABORT | KW_ADD | KW_ADMIN | KW_AFTER | KW_ANALYZE | KW_ARCHIVE | KW_ASC | KW_BEFORE | KW_BUCKET | KW_BUCKETS | ||
| KW_CASCADE | KW_CBO | KW_CHANGE | KW_CHECK | KW_CLUSTER | KW_CLUSTERED | KW_CLUSTERSTATUS | KW_COLLECTION | KW_COLUMNS | ||
| KW_COMMENT | KW_COMPACT | KW_COMPACTIONS | KW_COMPUTE | KW_CONCATENATE | KW_CONTINUE | KW_COST | KW_DATA | KW_DAY | KW_CATALOG | KW_CATALOGS | ||
| KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DCPROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY | ||
| KW_DATABASES | KW_DATETIME | KW_DBPROPERTIES | KW_DCPROPERTIES |KW_PROPERTIES | KW_DEFERRED | KW_DEFINED | KW_DELIMITED | KW_DEPENDENCY | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: space |
||
| KW_DESC | KW_DIRECTORIES | KW_DIRECTORY | KW_DISABLE | KW_DISTRIBUTE | KW_DISTRIBUTED | KW_DOW | KW_ELEM_TYPE | ||
| KW_ENABLE | KW_ENFORCED | KW_ESCAPED | KW_EXCLUSIVE | KW_EXPLAIN | KW_EXPORT | KW_FIELDS | KW_FILE | KW_FILEFORMAT | ||
| KW_FIRST | KW_FORMAT | KW_FORMATTED | KW_FUNCTIONS | KW_HOUR | KW_IDXPROPERTIES | KW_RESPECT | KW_IGNORE | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties; | ||
|
||
import org.apache.hadoop.hive.ql.QueryState; | ||
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; | ||
import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogAnalyzer; | ||
import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogDesc; | ||
import org.apache.hadoop.hive.ql.parse.ASTNode; | ||
import org.apache.hadoop.hive.ql.parse.HiveParser; | ||
import org.apache.hadoop.hive.ql.parse.SemanticException; | ||
|
||
import java.util.Map; | ||
|
||
/** | ||
* Analyzer for catalog set properties commands. | ||
*/ | ||
@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERCATALOG_PROPERTIES) | ||
public class AlterCatalogSetPropertiesAnalyzer extends AbstractAlterCatalogAnalyzer { | ||
public AlterCatalogSetPropertiesAnalyzer(QueryState queryState) throws SemanticException { | ||
super(queryState); | ||
} | ||
|
||
@Override | ||
protected AbstractAlterCatalogDesc buildAlterCatalogDesc(ASTNode root) throws SemanticException { | ||
String catalogName = unescapeIdentifier(root.getChild(0).getText()); | ||
|
||
Map<String, String> catProps = null; | ||
for (int i = 1; i < root.getChildCount(); i++) { | ||
ASTNode childNode = (ASTNode) root.getChild(i); | ||
if (childNode.getToken().getType() == HiveParser.TOK_CATALOGPROPERTIES) { | ||
catProps = getProps((ASTNode) childNode.getChild(0)); | ||
break; | ||
} else { | ||
throw new SemanticException("Unrecognized token in ALTER CATALOG statement"); | ||
} | ||
} | ||
|
||
return new AlterCatalogSetPropertiesDesc(catalogName, catProps); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties; | ||
|
||
import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogDesc; | ||
import org.apache.hadoop.hive.ql.plan.Explain; | ||
|
||
import java.util.Map; | ||
|
||
/** | ||
* DDL task description for ALTER CATALOG ... SET PROPERTIES ... commands. | ||
*/ | ||
@Explain(displayName = "Set Catalog Properties", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED }) | ||
public class AlterCatalogSetPropertiesDesc extends AbstractAlterCatalogDesc { | ||
private static final long serialVersionUID = 1L; | ||
|
||
private final Map<String, String> catProperties; | ||
|
||
public AlterCatalogSetPropertiesDesc(String catalogName, Map<String, String> catProperties) { | ||
super(catalogName); | ||
this.catProperties = catProperties; | ||
} | ||
|
||
@Explain(displayName="properties") | ||
public Map<String, String> getCatalogProperties() { | ||
return catProperties; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.hadoop.hive.ql.ddl.catalog.alter.properties; | ||
|
||
import org.apache.hadoop.hive.metastore.api.Catalog; | ||
import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; | ||
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory; | ||
import org.apache.hadoop.hive.ql.ddl.catalog.alter.AbstractAlterCatalogOperation; | ||
import org.apache.hadoop.hive.ql.metadata.HiveException; | ||
import org.apache.hadoop.hive.ql.parse.HiveParser; | ||
|
||
import java.util.Map; | ||
|
||
/** | ||
* Analyzer for catalog set properties commands. | ||
*/ | ||
@DDLSemanticAnalyzerFactory.DDLType(types = HiveParser.TOK_ALTERCATALOG_PROPERTIES) | ||
public class AlterCatalogSetPropertiesOperation | ||
extends AbstractAlterCatalogOperation<AlterCatalogSetPropertiesDesc> { | ||
public AlterCatalogSetPropertiesOperation(DDLOperationContext context, AlterCatalogSetPropertiesDesc desc) { | ||
super(context, desc); | ||
} | ||
|
||
@Override | ||
protected void doAlteration(Catalog catalog) throws HiveException { | ||
Map<String, String> newParams = desc.getCatalogProperties(); | ||
Map<String, String> params = catalog.getParameters(); | ||
|
||
// if both old and new params are not null, merge them | ||
if (params != null && newParams != null) { | ||
params.putAll(newParams); | ||
catalog.setParameters(params); | ||
} else { | ||
// if one of them is null, replace the old params with the new one | ||
catalog.setParameters(newParams); | ||
} | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,6 +22,7 @@ | |
import org.apache.hadoop.hive.ql.plan.Explain; | ||
|
||
import java.io.Serializable; | ||
import java.util.Map; | ||
|
||
/** | ||
* DDL task description for CREATE CATALOG commands. | ||
|
@@ -34,12 +35,14 @@ public class CreateCatalogDesc implements DDLDesc, Serializable { | |
private final String comment; | ||
private final String locationUri; | ||
private final boolean ifNotExists; | ||
private final Map<String, String> catProperties; | ||
|
||
public CreateCatalogDesc(String catalogName, String comment, String locationUri, boolean ifNotExists) { | ||
public CreateCatalogDesc(String catalogName, String comment, String locationUri, boolean ifNotExists, Map<String, String> dcProperties) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what is |
||
this.catalogName = catalogName; | ||
this.comment = comment; | ||
this.locationUri = locationUri; | ||
this.ifNotExists = ifNotExists; | ||
this.catProperties = dcProperties; | ||
} | ||
|
||
@Explain(displayName="name", explainLevels = { Explain.Level.USER, Explain.Level.DEFAULT, Explain.Level.EXTENDED }) | ||
|
@@ -61,4 +64,8 @@ public String getLocationUri() { | |
public boolean isIfNotExists() { | ||
return ifNotExists; | ||
} | ||
|
||
public Map<String, String> getCatlogProperties() { | ||
return catProperties; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,7 +19,7 @@ DROP CATALOG test_cat; | |
SHOW CATALOGS; | ||
|
||
-- CREATE INE doesn't exist | ||
CREATE CATALOG IF NOT EXISTS test_cat LOCATION '/tmp/test_cat' COMMENT 'Hive test catalog'; | ||
CREATE CATALOG IF NOT EXISTS test_cat LOCATION '/tmp/test_cat' COMMENT 'Hive test catalog' PROPERTIES('key1'='value1');; | ||
SHOW CATALOGS; | ||
|
||
-- DROP IE exists | ||
|
@@ -45,3 +45,7 @@ SHOW CATALOGS LIKE 'test__'; | |
-- ALTER LOCATION | ||
ALTER CATALOG test_cat SET LOCATION '/tmp/test_cat_new'; | ||
DESC CATALOG EXTENDED test_cat; | ||
|
||
-- ALTER PROPERTIES. | ||
-- TODO catalog. Check the catalog's properties after we implement 'desc formatted' or 'show create catalog'. | ||
ALTER CATALOG test_cat SET PROPERTIES ('key2'='value2'); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do we have support for describe catalog or not yet? |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why is this change?