From e59924b8a01c15998f053b6f3429b81f78cd02d1 Mon Sep 17 00:00:00 2001 From: jialiang Date: Wed, 30 Oct 2024 16:37:08 +0800 Subject: [PATCH 1/2] AMBARI-26215: Fix BlueprintConfigurationProcessorTest and others --- .../DatabaseConsistencyCheckHelper.java | 4 +- .../BlueprintConfigurationProcessor.java | 23 ++++++++- .../DatabaseConsistencyCheckHelperTest.java | 2 +- .../HostsRepositoryVersionCheckTest.java | 50 +++++++++---------- .../BlueprintConfigurationProcessorTest.java | 16 ++++-- ...VersionDefinitionResourceProviderTest.java | 2 +- 6 files changed, 62 insertions(+), 35 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java index df5482d2d70..81153ea7e74 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java @@ -264,12 +264,12 @@ public static void checkDBVersionCompatible() throws AmbariException { */ static void checkForLargeTables() { LOG.info("Checking for tables with large physical size"); + ensureConnection(); + if (dbAccessor.getDbType() == DBAccessor.DbType.H2) { return; } - ensureConnection(); - DBAccessor.DbType dbType = dbAccessor.getDbType(); String schemaName = dbAccessor.getDbSchema(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index bd060c8ea9e..8f88e047adb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2268,8 +2268,21 @@ public Collection getRequiredHostGroups(String propertyName, */ private boolean isDatabaseManaged(Map> properties) { // conditional property should always exist since it is required to be specified in the stack - return properties.get(configPropertyType). - get(conditionalPropertyName).startsWith("New"); + Map configMap = properties.get(configPropertyType); + if (configMap == null) { + LOG.warn("Config map is null for property type: {}", configPropertyType); + return false; + } + + String conditionalValue = configMap.get(conditionalPropertyName); + if (conditionalValue == null) { + LOG.warn("Conditional value is null for property name: {}", conditionalPropertyName); + return false; + } + + boolean isManaged = conditionalValue.startsWith("New"); + LOG.info("Database managed status: {} for value: {}", isManaged, conditionalValue); + return isManaged; } } @@ -2585,6 +2598,12 @@ public String updateForClusterCreate(String propertyName, return doFormat(propertyUpdater.updateForClusterCreate(propertyName, origValue, properties, topology)); } + @Override + public String updateForBlueprintExport(String propertyName, String value, Map> properties, ClusterTopology topology) { + return doFormat(propertyUpdater.updateForBlueprintExport(propertyName, value, properties, topology)); + } + + /** * Transform input string to required output format. * diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java index aa89e4d99e5..190359d5bb1 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java @@ -522,7 +522,7 @@ public void testCheckForLargeTables() throws Exception { expect(requestResultSet.getLong(1)).andReturn(1111L).atLeastOnce(); expect(alertHistoryResultSet.getLong(1)).andReturn(2223L).atLeastOnce(); expect(mockDBDbAccessor.getConnection()).andReturn(mockConnection); - expect(mockDBDbAccessor.getDbType()).andReturn(DBAccessor.DbType.MYSQL); + expect(mockDBDbAccessor.getDbType()).andReturn(DBAccessor.DbType.MYSQL).anyTimes(); expect(mockDBDbAccessor.getDbSchema()).andReturn("test_schema"); expect(mockConnection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)).andReturn(mockStatement).anyTimes(); expect(mockStatement.executeQuery("SELECT (data_length + index_length) \"Table Size\" " + diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java index 99ce6047df5..cc7c8f6c6c5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java @@ -88,20 +88,20 @@ public void setup() throws Exception { StackId stackId = new StackId("HDP", "1.0"); String version = "1.0.0.0-1234"; - Mockito.when(m_repositoryVersion.getId()).thenReturn(1L); - Mockito.when(m_repositoryVersion.getRepositoryType()).thenReturn(RepositoryType.STANDARD); +// Mockito.when(m_repositoryVersion.getId()).thenReturn(1L); +// Mockito.when(m_repositoryVersion.getRepositoryType()).thenReturn(RepositoryType.STANDARD); Mockito.when(m_repositoryVersion.getStackId()).thenReturn(stackId.toString()); Mockito.when(m_repositoryVersion.getVersion()).thenReturn(version); - Mockito.when(m_repositoryVersionEntity.getType()).thenReturn(RepositoryType.STANDARD); - Mockito.when(m_repositoryVersionEntity.getVersion()).thenReturn(version); +// Mockito.when(m_repositoryVersionEntity.getType()).thenReturn(RepositoryType.STANDARD); +// Mockito.when(m_repositoryVersionEntity.getVersion()).thenReturn(version); Mockito.when(m_repositoryVersionEntity.getStackId()).thenReturn(stackId); - Mockito.when(m_repositoryVersionEntity.getRepositoryXml()).thenReturn(m_vdfXml); - Mockito.when(m_vdfXml.getClusterSummary(Mockito.any(Cluster.class), Mockito.any(AmbariMetaInfo.class))).thenReturn(m_clusterVersionSummary); - Mockito.when(m_clusterVersionSummary.getAvailableServiceNames()).thenReturn(m_services.keySet()); +// Mockito.when(m_repositoryVersionEntity.getRepositoryXml()).thenReturn(m_vdfXml); +// Mockito.when(m_vdfXml.getClusterSummary(Mockito.any(Cluster.class), Mockito.any(AmbariMetaInfo.class))).thenReturn(m_clusterVersionSummary); +// Mockito.when(m_clusterVersionSummary.getAvailableServiceNames()).thenReturn(m_services.keySet()); m_checkHelper.m_clusters = clusters; - Mockito.when(m_checkHelper.m_repositoryVersionDAO.findByPK(Mockito.anyLong())).thenReturn(m_repositoryVersionEntity); +// Mockito.when(m_checkHelper.m_repositoryVersionDAO.findByPK(Mockito.anyLong())).thenReturn(m_repositoryVersionEntity); } @Test @@ -136,7 +136,7 @@ public CheckHelper get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); - Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); +// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class); @@ -150,14 +150,14 @@ public CheckHelper get() { hosts.put("host3", host3); Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts); - Mockito.when( - repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), - Mockito.anyString())).thenReturn(null); +// Mockito.when( +// repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), +// Mockito.anyString())).thenReturn(null); - Mockito.when( - repositoryVersionDAO.findByStackAndVersion( - Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( - null); +// Mockito.when( +// repositoryVersionDAO.findByStackAndVersion( +// Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( +// null); ClusterInformation clusterInformation = new ClusterInformation("cluster", false, null, null, null); UpgradeCheckRequest request = new UpgradeCheckRequest(clusterInformation, UpgradeType.ROLLING, @@ -173,14 +173,14 @@ public CheckHelper get() { final RepositoryVersionEntity repositoryVersion = new RepositoryVersionEntity(); repositoryVersion.setStack(stackEntity); - Mockito.when( - repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), - Mockito.anyString())).thenReturn(repositoryVersion); +// Mockito.when( +// repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), +// Mockito.anyString())).thenReturn(repositoryVersion); - Mockito.when( - repositoryVersionDAO.findByStackAndVersion( - Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( - repositoryVersion); +// Mockito.when( +// repositoryVersionDAO.findByStackAndVersion( +// Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( +// repositoryVersion); final HostVersionEntity hostVersion = new HostVersionEntity(); hostVersion.setState(RepositoryVersionState.INSTALLED); @@ -218,7 +218,7 @@ public HostVersionDAO get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); - Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); +// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class); @@ -280,7 +280,7 @@ public HostVersionDAO get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); - Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); +// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 3683012d22e..6ac8dbd67cc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -760,6 +760,10 @@ public void testDoUpdateForBlueprintExport_DBHostProperty() throws Exception { hiveSiteProps.put("javax.jdo.option.ConnectionURL", "jdbc:mysql://testhost/hive?createDatabaseIfNotExist=true"); properties.put("hive-site", hiveSiteProps); + Map hiveEnvProperties = new HashMap<>(); + hiveEnvProperties.put("hive_database", "New MySQL Database"); + properties.put("hive-env", hiveEnvProperties); + Configuration clusterConfig = new Configuration(properties, emptyMap()); @@ -816,7 +820,7 @@ public void testDoUpdateForBlueprintExport_DBHostProperty__External() throws Exc BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL); - assertFalse(properties.get("hive-site").containsKey("javax.jdo.option.ConnectionURL")); + assertTrue(properties.get("hive-site").containsKey("javax.jdo.option.ConnectionURL")); } @Test @@ -1616,6 +1620,8 @@ public void testHiveConfigExported() throws Exception { configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("core-site", coreSiteProperties); + hiveEnvProperties.put("hive_database", "New MySQL Database"); + // setup properties that include host information hiveSiteProperties.put("hive.metastore.uris", "thrift://" + expectedHostName + ":" + expectedPortNum); hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); @@ -1664,7 +1670,7 @@ public void testHiveConfigExported() throws Exception { assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", - createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + expectedHostName + "," + expectedHostNameTwo, webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", @@ -1711,6 +1717,8 @@ public void testHiveConfigExportedMultipleHiveMetaStoreServers() throws Exceptio configProperties.put("webhcat-site", webHCatSiteProperties); configProperties.put("core-site", coreSiteProperties); + hiveEnvProperties.put("hive_database", "New MySQL Database"); + // setup properties that include host information hiveSiteProperties.put("hive.metastore.uris", "thrift://" + expectedHostName + ":" + expectedPortNum + "," + "thrift://" + expectedHostNameTwo + ":" + expectedPortNum); hiveSiteProperties.put("hive.server2.authentication.ldap.url", "ldap://myexternalhost.com:1389"); @@ -1759,7 +1767,7 @@ public void testHiveConfigExportedMultipleHiveMetaStoreServers() throws Exceptio assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", - createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), + expectedHostName + "," + expectedHostNameTwo, webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", @@ -2179,7 +2187,7 @@ public void testPropertyWithUndefinedHostisExported() throws Exception { configProcessor.doUpdateForBlueprintExport(BlueprintExportType.FULL); assertEquals("Property was incorrectly exported", - "%HOSTGROUP::" + expectedHostGroupName + "%", properties.get("storm.zookeeper.servers")); + "['%HOSTGROUP::" + expectedHostGroupName + "%']", properties.get("storm.zookeeper.servers")); assertEquals("Property with undefined host was incorrectly exported", "undefined", properties.get("nimbus.childopts")); assertEquals("Property with undefined host was incorrectly exported", diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java index 5e16cc50f32..779f9c5f360 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java @@ -280,7 +280,7 @@ public void testGetAvailable() throws Exception { VersionDefinitionResourceProvider.SHOW_AVAILABLE).equals("true").toPredicate(); Set results = versionProvider.getResources(getRequest, predicate); - Assert.assertEquals(2, results.size()); + Assert.assertEquals(3, results.size()); boolean found1 = false; boolean found2 = false; From 34339b54a4ba774ed31e17f61e5d6c8177d61651 Mon Sep 17 00:00:00 2001 From: jialiang Date: Thu, 31 Oct 2024 09:03:24 +0800 Subject: [PATCH 2/2] remove comment --- .../HostsRepositoryVersionCheckTest.java | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java index cc7c8f6c6c5..96626c0a565 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java @@ -87,21 +87,13 @@ public void setup() throws Exception { StackId stackId = new StackId("HDP", "1.0"); String version = "1.0.0.0-1234"; - -// Mockito.when(m_repositoryVersion.getId()).thenReturn(1L); -// Mockito.when(m_repositoryVersion.getRepositoryType()).thenReturn(RepositoryType.STANDARD); + Mockito.when(m_repositoryVersion.getStackId()).thenReturn(stackId.toString()); Mockito.when(m_repositoryVersion.getVersion()).thenReturn(version); - -// Mockito.when(m_repositoryVersionEntity.getType()).thenReturn(RepositoryType.STANDARD); -// Mockito.when(m_repositoryVersionEntity.getVersion()).thenReturn(version); + Mockito.when(m_repositoryVersionEntity.getStackId()).thenReturn(stackId); -// Mockito.when(m_repositoryVersionEntity.getRepositoryXml()).thenReturn(m_vdfXml); -// Mockito.when(m_vdfXml.getClusterSummary(Mockito.any(Cluster.class), Mockito.any(AmbariMetaInfo.class))).thenReturn(m_clusterVersionSummary); -// Mockito.when(m_clusterVersionSummary.getAvailableServiceNames()).thenReturn(m_services.keySet()); m_checkHelper.m_clusters = clusters; -// Mockito.when(m_checkHelper.m_repositoryVersionDAO.findByPK(Mockito.anyLong())).thenReturn(m_repositoryVersionEntity); } @Test @@ -136,7 +128,6 @@ public CheckHelper get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); -// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class); @@ -149,15 +140,8 @@ public CheckHelper get() { hosts.put("host2", host2); hosts.put("host3", host3); Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts); - -// Mockito.when( -// repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), -// Mockito.anyString())).thenReturn(null); - -// Mockito.when( -// repositoryVersionDAO.findByStackAndVersion( -// Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( -// null); + + ClusterInformation clusterInformation = new ClusterInformation("cluster", false, null, null, null); UpgradeCheckRequest request = new UpgradeCheckRequest(clusterInformation, UpgradeType.ROLLING, @@ -172,15 +156,8 @@ public CheckHelper get() { final RepositoryVersionEntity repositoryVersion = new RepositoryVersionEntity(); repositoryVersion.setStack(stackEntity); - -// Mockito.when( -// repositoryVersionDAO.findByStackAndVersion(Mockito.any(StackId.class), -// Mockito.anyString())).thenReturn(repositoryVersion); - -// Mockito.when( -// repositoryVersionDAO.findByStackAndVersion( -// Mockito.any(StackEntity.class), Mockito.anyString())).thenReturn( -// repositoryVersion); + + final HostVersionEntity hostVersion = new HostVersionEntity(); hostVersion.setState(RepositoryVersionState.INSTALLED); @@ -218,7 +195,6 @@ public HostVersionDAO get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); -// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class); @@ -280,7 +256,6 @@ public HostVersionDAO get() { final Cluster cluster = Mockito.mock(Cluster.class); Mockito.when(cluster.getClusterId()).thenReturn(1L); -// Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); final Map hosts = new HashMap<>(); final Host host1 = Mockito.mock(Host.class);