Merge pull request #1860 from HubSpot/y_u_load_so_slow

Make SingularityUI snappier

SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java

@@ -638,6 +638,18 @@ public List<SingularityRequestParent> getActiveRequests(
         user, valueOrFalse(filterRelevantForUser), valueOrFalse(includeFullRequestData), Optional.fromNullable(limit), requestTypes);
   }
 
+  @GET
+  @Path("/ids")
+  @Operation(summary = "Retrieve the list of active request ids")
+  public List<String> getActiveRequests(
+      @Parameter(hidden = true) @Auth SingularityUser user,
+      @Parameter(description = "Fetched a cached version of this data to limit expensive operations") @QueryParam("useWebCache") Boolean useWebCache) {
+    return filterAutorized(Lists.newArrayList(requestManager.getRequests(useWebCache(useWebCache))), SingularityAuthorizationScope.READ, user)
+        .stream()
+        .map((r) -> r.getRequest().getId())
+        .collect(Collectors.toList());
+  }
+
 
   @GET
   @PropertyFiltering

SingularityUI/app/actions/api/history.es6

@@ -78,7 +78,10 @@ export const FetchTaskSearchParams = buildApiAction(
     } else {
       url = `/history/tasks?count=${count}&page=${page}&${Utils.queryParams(args)}`;
     }
-    return { url };
+    return {
+      url,
+      catchStatusCodes: [404, 500]
+    };
   });
 
 export const FetchRequestRunHistory = buildApiAction(

SingularityUI/app/actions/api/requests.es6

@@ -5,26 +5,26 @@ export const FetchRequests = buildApiAction(
   {url: '/requests'}
 );
 
-export const FetchUserRelevantRequests = buildApiAction(
-  'FETCH_USER_RELEVANT_REQUESTS',
-  (requestTypes = []) => {
-    let params = 'filterRelevantForUser=true&includeFullRequestData=true&limit=50'
-    _.each(requestTypes, (requestType) => {
-      params = params + `&requestType=${requestType}`
-    });
-    return {url: `/requests?${params}`}
-  }
+export const FetchRequestIds = buildApiAction(
+  'FETCH_REQUESTS',
+  {url: '/requests/ids?useWebCache=true'}
 );
 
 export const FetchRequestsInState = buildApiAction(
   'FETCH_REQUESTS_IN_STATE',
-  (state, renderNotFoundIf404) => {
+  (state = 'all', renderNotFoundIf404 = true, propertyFilter = null) => {
+    let queryString = '?useWebCache=true';
+    const propertyJoin = '&property=';
+    if (propertyFilter != null) {
+      queryString += '&property=';
+      queryString += propertyFilter.join(propertyJoin);
+    }
     if (_.contains(['pending', 'cleanup'], state)) {
-      return {url: `/requests/queued/${state}`, renderNotFoundIf404};
+      return {url: `/requests/queued/${state}`, renderNotFoundIf404}; // no property filter for these, different format
     } else if (_.contains(['all', 'noDeploy', 'activeDeploy', 'overUtilizedCpu', 'underUtilizedCpu', 'underUtilizedMem', 'underUtilizedDisk'], state)) {
-      return {url: '/requests', renderNotFoundIf404};
+      return {url: `/requests${queryString}`, renderNotFoundIf404};
     }
-    return {url: `/requests/${state}`, renderNotFoundIf404};
+    return {url: `/requests/${state}${queryString}`, renderNotFoundIf404};
   }
 );
 

SingularityUI/app/actions/ui/requests.es6

@@ -1,9 +1,17 @@
-import { FetchRequestsInState } from '../../actions/api/requests';
+import { FetchRequestsInState, FetchRequestIds } from '../../actions/api/requests';
 import { FetchRequestUtilizations } from '../../actions/api/utilization';
 
-export const refresh = (state) => (dispatch) => {
-    const promises = []
-	promises.push(dispatch(FetchRequestsInState.trigger(state === 'cleaning' ? 'cleanup' : state, true)));
-	promises.push(dispatch(FetchRequestUtilizations.trigger()));
-	return Promise.all(promises);
+export const refresh = (state, propertyFilter) => (dispatch) => {
+  const promises = []
+  promises.push(dispatch(FetchRequestsInState.trigger(state === 'cleaning' ? 'cleanup' : state, true, propertyFilter)));
+  promises.push(dispatch(FetchRequestUtilizations.trigger()));
+  return Promise.all(promises);
+}
+
+export const initialize = (requestIds) => (dispatch) => {
+  if (requestIds.isFetching || requestIds.data.length) {
+    return Promise.resolve();
+  } else {
+    return dispatch(FetchRequestIds.trigger());
+  }
 }

SingularityUI/app/components/globalSearch/GlobalSearch.jsx

@@ -2,15 +2,15 @@ import React from 'react';
 import classNames from 'classnames';
 import { connect } from 'react-redux';
 import { withRouter } from 'react-router';
-import { FetchRequests } from '../../actions/api/requests';
+import { FetchRequestIds } from '../../actions/api/requests';
 import { SetVisibility } from '../../actions/ui/globalSearch';
 import { refresh } from '../../actions/ui/requestDetail';
 import { push } from 'react-router-redux';
 import { Link } from 'react-router';
 
 import { Typeahead } from 'react-typeahead';
 import key from 'keymaster';
-import filterSelector from '../../selectors/requests/filterSelector';
+import idSelector from '../../selectors/idSelector';
 
 class GlobalSearch extends React.Component {
 
@@ -72,18 +72,10 @@ class GlobalSearch extends React.Component {
   }
 
   searchOptions(inputValue, options) {
-    const searched = filterSelector({
-      requestsInState: options,
+    const searched = idSelector({
+      options: options,
       filter: {
-        state: 'all',
-        searchFilter: inputValue,
-        subFilter: [
-          'SERVICE',
-          'WORKER',
-          'SCHEDULED',
-          'ON_DEMAND',
-          'RUN_ONCE'
-        ]
+        searchFilter: inputValue
       }
     });
 
@@ -111,10 +103,8 @@ class GlobalSearch extends React.Component {
   }
 
   render() {
-    const options = _.map(this.props.requests, (requestParent) => ({
-      request: requestParent.request,
-      id: requestParent.request.id,
-      requestDeployState: requestParent.requestDeployState
+    const options = _.map(this.props.requests, (id) => ({
+      id: id,
     }));
 
     const globalSearchClasses = classNames('global-search', {
@@ -141,8 +131,8 @@ class GlobalSearch extends React.Component {
               }}
               placeholder="Search all requests"
               onOptionSelected={this.optionSelected}
-              searchOptions={this.searchOptions}
               displayOption={this.renderOption}
+              searchOptions={this.searchOptions}
               formInputOption={this.getValueFromOption}
               inputDisplayOption={this.getValueFromOption}
             />
@@ -155,10 +145,10 @@ class GlobalSearch extends React.Component {
 }
 
 export default connect((state) => ({
-  requests: state.api.requests.data,
+  requests: state.api.requestIds.data,
   visible: state.ui.globalSearch.visible
 }), {
-  getRequests: FetchRequests.trigger,
+  getRequests: FetchRequestIds.trigger,
   setVisibility: SetVisibility,
   push,
   refresh

SingularityUI/app/components/requests/RequestsPage.jsx

@@ -14,19 +14,22 @@ import {
 } from '../../actions/api/requests';
 import { FetchRequestRunHistory } from '../../actions/api/history';
 import { FetchTaskFiles } from '../../actions/api/sandbox';
-import { refresh } from '../../actions/ui/requests';
+import { refresh, initialize } from '../../actions/ui/requests';
 
 import UITable from '../common/table/UITable';
 import RequestFilters from './RequestFilters';
 import * as Cols from './Columns';
 
 import filterSelector from '../../selectors/requests/filterSelector';
+import idSelector from '../../selectors/idSelector';
 
 import Utils from '../../utils';
 import Loader from '../common/Loader';
 
 class RequestsPage extends Component {
 
+  static propertyFilter = ['state', 'request.id', 'request.requestType', 'request.instances', 'request.group', 'request.bounceAfterScale', 'requestDeployState.activeDeploy.deployId', 'requestDeployState.activeDeploy.user', 'requestDeployState.activeDeploy.timestamp', 'requestDeployState.activeDeploy.loadBalancerOptions']
+
   static propTypes = {
     requestsInState: PropTypes.array,
     fetchFilter: PropTypes.func,
@@ -56,6 +59,10 @@ class RequestsPage extends Component {
     };
   }
 
+  componentWillMount() {
+    this.props.fetchFilter();
+  }
+
   handleFilterChange(filter) {
     const lastFilterState = this.props.filter.state;
     this.setState({
@@ -107,7 +114,17 @@ class RequestsPage extends Component {
   }
 
   render() {
-    const displayRequests = filterSelector({requestsInState: this.props.requestsInState, filter: this.props.filter, requestUtilizations: this.props.requestUtilizations});
+    let displayRequests = []
+    if (this.props.requestsInState.length) {
+      displayRequests = filterSelector({requestsInState: this.props.requestsInState, filter: this.props.filter, requestUtilizations: this.props.requestUtilizations});
+    } else if (this.props.requestIds.data.length) {
+      const options = _.map(this.props.requestIds.data, (id) => ({
+        id: id,
+        starred: _.contains(Utils.maybe(this.props.user, ['data', 'settings', 'starredRequestIds'], []), id) ? 1 : 0,
+        request: {id: id}
+      }));
+      displayRequests = idSelector({options: options, filter: this.props.filter}).sort((a, b) => b.starred - a.starred);
+    }
 
     let table;
     if (this.state.loading) {
@@ -120,6 +137,8 @@ class RequestsPage extends Component {
           ref="table"
           data={displayRequests}
           keyGetter={(request) => (request.request ? request.request.id : request.requestId)}
+          rowChunkSize={20}
+          paginated={true}
         >
           {this.getColumns()}
         </UITable>
@@ -167,14 +186,16 @@ function mapStateToProps(state, ownProps) {
     notFound: statusCode === 404,
     requestsInState: modifiedRequests,
     requestUtilizations: state.api.requestUtilizations.data,
+    requestIds: state.api.requestIds,
     groups: userGroups,
+    user: state.api.user,
     filter
   };
 }
 
-function mapDispatchToProps(dispatch) {
+function mapDispatchToProps(dispatch, ownProps) {
   return {
-    fetchFilter: (state) => dispatch(FetchRequestsInState.trigger(state === 'cleaning' ? 'cleanup' : state, true)),
+    fetchFilter: (state) => dispatch(FetchRequestsInState.trigger(state === 'cleaning' ? 'cleanup' : state, true, RequestsPage.propertyFilter)),
     removeRequest: (requestid, data) => dispatch(RemoveRequest.trigger(requestid, data)),
     unpauseRequest: (requestId, data) => dispatch(UnpauseRequest.trigger(requestId, data)),
     runNow: (requestId, data) => dispatch(RunRequest.trigger(requestId, data)),
@@ -188,4 +209,4 @@ function mapDispatchToProps(dispatch) {
   };
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(rootComponent(withRouter(RequestsPage), (props) => refresh(props.params.state || 'all'), true, false));
+export default connect(mapStateToProps, mapDispatchToProps)(rootComponent(withRouter(RequestsPage), (props) => refresh(props.params.state || 'all', RequestsPage.propertyFilter), true, false, (props) => initialize(props.requestIds)));

SingularityUI/app/initialize.jsx

@@ -1,9 +1,3 @@
-import Raven from 'raven-js';
-
-if (window.config.sentryDsn) {
-  Raven.config(window.config.sentryDsn).install();
-}
-
 // explicit polyfills for older browsers
 import 'core-js/es6';
 
@@ -77,9 +71,6 @@ document.addEventListener('DOMContentLoaded', () => {
       if (!store.getState().api.user.data.user) {
         return renderUserIdForm();
       } else {
-        if (window.config.sentryDsn) {
-          Raven.setUserContext({ email: store.getState().api.user.data.user.email });
-        }
         userId = store.getState().api.user.data.user.id
         // Set up starred requests
         maybeImportStarredRequests(store, store.getState().api.user, userId);

SingularityUI/app/reducers/api/index.es6

@@ -40,7 +40,7 @@ import {
 
 import {
   FetchRequests,
-  FetchUserRelevantRequests,
+  FetchRequestIds,
   FetchRequest,
   SaveRequest,
   RemoveRequest,
@@ -116,9 +116,9 @@ const decommissionRack = buildApiActionReducer(DecommissionRack, []);
 const removeRack = buildApiActionReducer(RemoveRack, []);
 const reactivateRack = buildApiActionReducer(ReactivateRack, []);
 const request = buildKeyedApiActionReducer(FetchRequest);
+const requestIds = buildApiActionReducer(FetchRequestIds, [])
 const saveRequest = buildApiActionReducer(SaveRequest);
 const requests = buildApiActionReducer(FetchRequests, []);
-const userRelevantRequests = buildApiActionReducer(FetchUserRelevantRequests, []);
 const requestsInState = buildApiActionReducer(FetchRequestsInState, []);
 const requestHistory = buildKeyedApiActionReducer(FetchRequestHistory, []);
 const requestArgHistory = buildKeyedApiActionReducer(FetchRequestArgHistory, []);
@@ -150,7 +150,7 @@ const requestGroups = buildApiActionReducer(FetchGroups, []);
 const inactiveHosts = buildApiActionReducer(FetchInactiveHosts, []);
 const utilization = buildApiActionReducer(FetchUtilization, {});
 const requestUtilizations = buildApiActionReducer(FetchRequestUtilizations, []);
-const requestUtilization = buildKeyedApiActionReducer(FetchRequestUtilization, []);
+const requestUtilization = buildKeyedApiActionReducer(FetchRequestUtilization, {});
 
 export default combineReducers({
   user,
@@ -180,7 +180,7 @@ export default combineReducers({
   unpauseRequest,
   exitRequestCooldown,
   requests,
-  userRelevantRequests,
+  requestIds,
   requestsInState,
   requestHistory,
   requestArgHistory,

SingularityUI/app/reducers/ui/index.es6

@@ -3,13 +3,11 @@ import { combineReducers } from 'redux';
 import refresh from './refresh';
 import form from './form';
 import globalSearch from './globalSearch';
-import dashboard from './dashboard';
 import slaves from './slaves';
 
 export default combineReducers({
   refresh,
   form,
   globalSearch,
-  dashboard,
   slaves
 });

SingularityUI/app/selectors/idSelector.es6

@@ -0,0 +1,28 @@
+import { createSelector } from 'reselect';
+import micromatch from 'micromatch';
+import fuzzy from 'fuzzy';
+import Utils from '../utils';
+
+const getOptions = (state) => state.options;
+const getFilter = (state) => state.filter;
+
+export default createSelector([getOptions, getFilter], (options, filter) => {
+  let filteredOptions = options;
+  const getId = (parent) => parent.id || '';
+  if (Utils.isGlobFilter(filter.searchFilter)) {
+    const idMatches = _.filter(filteredOptions, (parent) => (
+      micromatch.isMatch(getId(parent), `${filter.searchFilter}*`)
+    ));
+    filteredOptions = idMatches;
+  } else if (filter.searchFilter) {
+    // Allow searching by the first letter of each word by applying same
+    // search heuristics to just the upper case characters of each option
+    const idMatches = fuzzy.filter(filter.searchFilter, filteredOptions, {
+      extract: Utils.isAllUpperCase(filter.searchFilter)
+        ? (parent) => Utils.getUpperCaseCharacters(getId(parent))
+        : getId,
+    });
+    filteredOptions = Utils.fuzzyFilter(filter.searchFilter, idMatches);
+  }
+  return filteredOptions;
+});