Skip to content

YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. #5030

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
goiri merged 6 commits into from
Oct 24, 2022
Merged

YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. #5030

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cda98cd
YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page.
Oct 14, 2022
bd620bf
Merge branch 'apache:trunk' into YARN-11345
slfan1989 Oct 17, 2022
0cf5fff
YARN-11345. Fix CheckStyle.
Oct 20, 2022
ca32701
YARN-11345. Fix CheckStyle.
Oct 20, 2022
d2b8206
YARN-11345. Fix CheckStyle.
Oct 21, 2022
23dc440
YARN-11345. Fix CheckStyle.
Oct 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...op-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public interface YarnWebParams {
String APP_STATE = "app.state";
String APP_START_TIME_BEGIN = "app.started-time.begin";
String APP_START_TIME_END = "app.started-time.end";
String APP_SC = "app.subcluster";
String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
Expand Down
179 changes: 128 additions & 51 deletions ...rn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,18 @@
import static org.apache.commons.text.StringEscapeUtils.escapeHtml4;
import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;

import com.sun.jersey.api.client.Client;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
Expand All @@ -34,34 +41,100 @@
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;

import com.google.inject.Inject;

import java.util.HashMap;
import java.util.Map;

/**
* Applications block for the Router Web UI.
*/
public class AppsBlock extends HtmlBlock {
public class AppsBlock extends RouterBlock {

private final Router router;
private final Configuration conf;

@Inject
AppsBlock(Router router, ViewContext ctx) {
super(ctx);
super(router, ctx);
this.router = router;
this.conf = this.router.getConfig();
}

@Override
protected void render(Block html) {
// Get the applications from the Resource Managers
Configuration conf = this.router.getConfig();

boolean isEnabled = isYarnFederationEnabled();

// Get subClusterName
String subClusterName = $(APP_SC);
String reqState = $(APP_STATE);

// We will try to get the subClusterName.
// If the subClusterName is not empty,
// it means that we need to get the Node list of a subCluster.
AppsInfo appsInfo = null;
if (subClusterName != null && !subClusterName.isEmpty()) {
initSubClusterMetricsOverviewTable(html, subClusterName);
appsInfo = getSubClusterAppsInfo(subClusterName, reqState);
} else {
// Metrics Overview Table
html.__(MetricsOverviewTable.class);
appsInfo = getYarnFederationAppsInfo(isEnabled);
}

initYarnFederationAppsOfCluster(appsInfo, html);
}

private static String escape(String str) {
return escapeEcmaScript(escapeHtml4(str));
}

private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
if (isEnabled) {
String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
}
return null;
}

private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
try {
SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);

if (subClusterInfo != null) {
// Prepare webAddress
String webAddress = subClusterInfo.getRMWebServiceAddress();
String herfWebAppAddress = "";
if (webAddress != null && !webAddress.isEmpty()) {
herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
}
}
} catch (Exception e) {
LOG.error("get AppsInfo From SubCluster = {} error.", subCluster, e);
}
return null;
}

private AppsInfo getSubClusterAppsInfoByWebAddress(String webAddress, String states) {
Client client = RouterWebServiceUtil.createJerseyClient(conf);
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
Map<String, String[]> queryParams = new HashMap<>();
if (StringUtils.isNotBlank(states)) {
queryParams.put("states", new String[]{states});
}
AppsInfo apps = RouterWebServiceUtil
.genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
client);
.genericForward(webAddress, null, AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, queryParams, conf,
client);
client.destroy();
return apps;
}

setTitle("Applications");
private void initYarnFederationAppsOfCluster(AppsInfo appsInfo, Block html) {

TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
.tr()
Expand All @@ -81,53 +154,57 @@ protected void render(Block html) {

// Render the applications
StringBuilder appsTableData = new StringBuilder("[\n");
for (AppInfo app : apps.getApps()) {
try {

String percent = String.format("%.1f", app.getProgress() * 100.0F);
String trackingURL =
app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsTableData.append("[\"")
.append("<a href='").append(trackingURL).append("'>")
.append(app.getAppId()).append("</a>\",\"")
.append(escape(app.getUser())).append("\",\"")
.append(escape(app.getName())).append("\",\"")
.append(escape(app.getApplicationType())).append("\",\"")
.append(escape(app.getQueue())).append("\",\"")
.append(String.valueOf(app.getPriority())).append("\",\"")
.append(app.getStartTime()).append("\",\"")
.append(app.getFinishTime()).append("\",\"")
.append(app.getState()).append("\",\"")
.append(app.getFinalStatus()).append("\",\"")
// Progress bar
.append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='")
.append(join(percent, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", percent, '%')).append("'> </div> </div>")
// History link
.append("\",\"<a href='").append(trackingURL).append("'>")
.append("History").append("</a>");
appsTableData.append("\"],\n");

} catch (Exception e) {
LOG.info(
"Cannot add application {}: {}", app.getAppId(), e.getMessage());

if (appsInfo != null && CollectionUtils.isNotEmpty(appsInfo.getApps())) {
for (AppInfo app : appsInfo.getApps()) {
try {

String percent = String.format("%.1f", app.getProgress() * 100.0F);
String trackingURL =
app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();

// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsTableData.append("[\"")
.append("<a href='").append(trackingURL).append("'>")
.append(app.getAppId()).append("</a>\",\"")
.append(escape(app.getUser())).append("\",\"")
.append(escape(app.getName())).append("\",\"")
.append(escape(app.getApplicationType())).append("\",\"")
.append(escape(app.getQueue())).append("\",\"")
.append(app.getPriority()).append("\",\"")
.append(app.getStartTime()).append("\",\"")
.append(app.getFinishTime()).append("\",\"")
.append(app.getState()).append("\",\"")
.append(app.getFinalStatus()).append("\",\"")
// Progress bar
.append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='")
.append(join(percent, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", percent, '%')).append("'> </div> </div>")
// History link
.append("\",\"<a href='").append(trackingURL).append("'>")
.append("History").append("</a>");
appsTableData.append("\"],\n");

} catch (Exception e) {
LOG.info("Cannot add application {}: {}", app.getAppId(), e.getMessage());
}
}

// The purpose of this part of the code is to remove redundant commas.

@goiri goiri Oct 20, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We know how many apps there will be, we should only add it if not the last:

if (appsInfo != null) {
  Collection<AppInfo> apps = appsInfo.getApps();
  if (CollectionUtils.isNotEmpty(apps)) {
    int numApps = apps.size();
    for (AppInfo app: apps) {
      ...
      if (i < numApps - 1) {
        appsTableData.append(",");
      }
    }
  }
}

@slfan1989 slfan1989 Oct 20, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your suggestion, I will refactor this part of the code.

// If we find that the second-to-last character is a comma, we need to delete this comma.
int appsTableDataLength = appsTableData.length();
char secondToLastChar = appsTableData.charAt(appsTableDataLength - 2);
if (secondToLastChar == ',') {
appsTableData.delete(appsTableDataLength - 2, appsTableDataLength - 1);
}
}
if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
appsTableData.delete(appsTableData.length() - 2,
appsTableData.length() - 1);
}

appsTableData.append("]");
html.script().$type("text/javascript")
.__("var appsTableData=" + appsTableData).__();

tbody.__().__();
}

private static String escape(String str) {
return escapeEcmaScript(escapeHtml4(str));
}
}
11 changes: 9 additions & 2 deletions ...arn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@

import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.webapp.SubView;

class AppsPage extends RouterView {
Expand All @@ -37,9 +39,14 @@ protected void preHead(Page.HTML<__> html) {
setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");

// Set the correct title.
String subClusterName = $(APP_SC);
String reqState = $(APP_STATE);
reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
setTitle(sjoin(reqState, "Applications"));

if(StringUtils.isBlank(subClusterName)){
subClusterName = "Federation ";
}
reqState = (StringUtils.isBlank(reqState) ? "All" : reqState);
setTitle(sjoin(subClusterName, reqState, "Applications"));
}

private String appsTableInit() {
Expand Down
21 changes: 19 additions & 2 deletions ...arn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@

import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;

import java.util.List;

/**
* Navigation block for the Router Web UI.
*/
Expand Down Expand Up @@ -60,7 +60,24 @@ public void render(Block html) {
subAppsList1.__().__();

// ### applications info
mainList.li().a(url("apps"), "Applications").__();
Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> subAppsList2 =
mainList.li().a(url("apps"), "Applications").ul();

subAppsList2.li().__();
for (String subClusterId : subClusterIds) {
Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>> subAppsList3 = subAppsList2.
li().a(url("apps", subClusterId), subClusterId);
Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>>> subAppsList4 =
subAppsList3.ul().$style("padding:0.3em 1em 0.1em 2em");
subAppsList4.li().__();
for (YarnApplicationState state : YarnApplicationState.values()) {
subAppsList4.
li().a(url("apps", subClusterId, state.toString()), state.toString()).__();
}
subAppsList4.li().__().__();
subAppsList3.__();
}
subAppsList2.__().__();

// ### tools
Hamlet.DIV<Hamlet> sectionBefore = mainList.__();
Expand Down
1 change: 1 addition & 0 deletions ...n-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) {
.genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
client);
client.destroy();
return nodes;
}

Expand Down
2 changes: 1 addition & 1 deletion ...server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public void setup() {
route("/", RouterController.class);
route("/cluster", RouterController.class, "about");
route("/about", RouterController.class, "about");
route("/apps", RouterController.class, "apps");
route(pajoin("/apps", APP_SC, APP_STATE), RouterController.class, "apps");
route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
route("/federation", RouterController.class, "federation");
}
Expand Down
18 changes: 18 additions & 0 deletions ...outer/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,4 +81,22 @@ public void testFederationNodeViewNotEnable()
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
}

@Test
public void testFederationAppViewEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
}

@Test
public void testFederationAppViewNotEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Not Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
}
}