[Console] Fix autocomplete_entities API crash when response size is too big (elastic#140569)

Fixes elastic#144310

### Summary

This PR addresses the issue of the Kibana instance restarting when the
response size is too big for the `autocomplete_entities` API. This
happens when a cluster has a large number of mappings and we try to
retrieve them all on the server side with
`esClient.asInternalUser.indices.getMapping()`. esClient does not handle
large responses well and throws an error that causes the Kibana instance
to restart. As node's max [string
length](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/length#description)
is 2^28-1 (~512MB) if the response size is over 512MB, it will throw an
error with the message "Invalid string length".

The fix is to use the raw http request to retrieve the mappings instead
of esClient and check the response size before sending it to the client.
If the response size is too big, we will return an empty object and log
the error in the server logs.

#### Proposed changes

- Remove ES JS client requests and use native Node.js HTTP client
instead
- Limit the response size to 10MB for the `autocomplete_entities` API

#### How to test this PR locally
To test this out, you will need to connect Kibana to a remote cluster
with a large number of mappings. We created a patch file that you can
apply to your local Kibana instance to test this PR. Since the patch
file contains credentials, we can't share it publicly. Please reach out
to me if you would like to test this PR. I will share the patch file and
the instructions to apply it.

Co-authored-by: Muhammad Ibragimov <muhammad.ibragimov@elastic.co>

Author: mibragimov

src/plugins/console/public/lib/autocomplete/components/component_template_autocomplete_component.js

@@ -6,12 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 export class ComponentTemplateAutocompleteComponent extends ListComponent {
   constructor(name, parent) {
-    super(name, getAutocompleteInfo().getEntityProvider('componentTemplates'), parent, true, true);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.COMPONENT_TEMPLATES),
+      parent,
+      true,
+      true
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/data_stream_autocomplete_component.js

@@ -7,11 +7,16 @@
  */
 
 import { ListComponent } from './list_component';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 
 export class DataStreamAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('dataStreams'), parent, multiValued);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.DATA_STREAMS),
+      parent,
+      multiValued
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/field_autocomplete_component.js

@@ -7,11 +7,11 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function FieldGenerator(context) {
-  return _.map(getAutocompleteInfo().getEntityProvider('fields', context), function (field) {
+  return _.map(getAutocompleteInfo().getEntityProvider(ENTITIES.FIELDS, context), function (field) {
     return { name: field.name, meta: field.type };
   });
 }

src/plugins/console/public/lib/autocomplete/components/index_autocomplete_component.js

@@ -7,7 +7,7 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function nonValidIndexType(token) {
@@ -16,7 +16,7 @@ function nonValidIndexType(token) {
 
 export class IndexAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued);
+    super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued);
   }
   validateTokens(tokens) {
     if (!this.multiValued && tokens.length > 1) {

src/plugins/console/public/lib/autocomplete/components/index_template_autocomplete_component.js

@@ -6,12 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 export class IndexTemplateAutocompleteComponent extends ListComponent {
   constructor(name, parent) {
-    super(name, getAutocompleteInfo().getEntityProvider('indexTemplates'), parent, true, true);
+    super(
+      name,
+      getAutocompleteInfo().getEntityProvider(ENTITIES.INDEX_TEMPLATES),
+      parent,
+      true,
+      true
+    );
   }
 
   getContextKey() {

src/plugins/console/public/lib/autocomplete/components/username_autocomplete_component.js

@@ -7,7 +7,7 @@
  */
 
 import _ from 'lodash';
-import { getAutocompleteInfo } from '../../../services';
+import { getAutocompleteInfo, ENTITIES } from '../../../services';
 import { ListComponent } from './list_component';
 
 function nonValidUsernameType(token) {
@@ -16,7 +16,7 @@ function nonValidUsernameType(token) {
 
 export class UsernameAutocompleteComponent extends ListComponent {
   constructor(name, parent, multiValued) {
-    super(name, getAutocompleteInfo().getEntityProvider('indices'), parent, multiValued);
+    super(name, getAutocompleteInfo().getEntityProvider(ENTITIES.INDICES), parent, multiValued);
   }
   validateTokens(tokens) {
     if (!this.multiValued && tokens.length > 1) {

src/plugins/console/public/lib/autocomplete_entities/types.ts

@@ -29,7 +29,7 @@ export interface FieldMapping {
   fields?: FieldMapping[];
 }
 
-export interface MappingsApiResponse {
+export interface AutoCompleteEntitiesApiResponse {
   mappings: IndicesGetMappingResponse;
   aliases: IndicesGetAliasResponse;
   dataStreams: IndicesGetDataStreamResponse;

src/plugins/console/public/services/autocomplete.ts

@@ -8,7 +8,7 @@
 
 import { createGetterSetter } from '@kbn/kibana-utils-plugin/public';
 import type { HttpSetup } from '@kbn/core/public';
-import type { MappingsApiResponse } from '../lib/autocomplete_entities/types';
+import type { AutoCompleteEntitiesApiResponse } from '../lib/autocomplete_entities/types';
 import { API_BASE_PATH } from '../../common/constants';
 import {
   Alias,
@@ -20,6 +20,15 @@ import {
 } from '../lib/autocomplete_entities';
 import { DevToolsSettings, Settings } from './settings';
 
+export enum ENTITIES {
+  INDICES = 'indices',
+  FIELDS = 'fields',
+  INDEX_TEMPLATES = 'indexTemplates',
+  COMPONENT_TEMPLATES = 'componentTemplates',
+  LEGACY_TEMPLATES = 'legacyTemplates',
+  DATA_STREAMS = 'dataStreams',
+}
+
 export class AutocompleteInfo {
   public readonly alias = new Alias();
   public readonly mapping = new Mapping();
@@ -39,19 +48,19 @@ export class AutocompleteInfo {
     context: { indices: string[]; types: string[] } = { indices: [], types: [] }
   ) {
     switch (type) {
-      case 'indices':
+      case ENTITIES.INDICES:
         const includeAliases = true;
         const collaborator = this.mapping;
         return () => this.alias.getIndices(includeAliases, collaborator);
-      case 'fields':
+      case ENTITIES.FIELDS:
         return this.mapping.getMappings(context.indices, context.types);
-      case 'indexTemplates':
+      case ENTITIES.INDEX_TEMPLATES:
         return () => this.indexTemplate.getTemplates();
-      case 'componentTemplates':
+      case ENTITIES.COMPONENT_TEMPLATES:
         return () => this.componentTemplate.getTemplates();
-      case 'legacyTemplates':
+      case ENTITIES.LEGACY_TEMPLATES:
         return () => this.legacyTemplate.getTemplates();
-      case 'dataStreams':
+      case ENTITIES.DATA_STREAMS:
         return () => this.dataStream.getDataStreams();
       default:
         throw new Error(`Unsupported type: ${type}`);
@@ -61,7 +70,7 @@ export class AutocompleteInfo {
   public retrieve(settings: Settings, settingsToRetrieve: DevToolsSettings['autocomplete']) {
     this.clearSubscriptions();
     this.http
-      .get<MappingsApiResponse>(`${API_BASE_PATH}/autocomplete_entities`, {
+      .get<AutoCompleteEntitiesApiResponse>(`${API_BASE_PATH}/autocomplete_entities`, {
         query: { ...settingsToRetrieve },
       })
       .then((data) => {
@@ -83,7 +92,7 @@ export class AutocompleteInfo {
     }
   }
 
-  private load(data: MappingsApiResponse) {
+  private load(data: AutoCompleteEntitiesApiResponse) {
     this.mapping.loadMappings(data.mappings);
     const collaborator = this.mapping;
     this.alias.loadAliases(data.aliases, collaborator);

src/plugins/console/public/services/index.ts

@@ -10,4 +10,9 @@ export { createHistory, History } from './history';
 export { createStorage, Storage, StorageKeys, setStorage, getStorage } from './storage';
 export type { DevToolsSettings } from './settings';
 export { createSettings, Settings, DEFAULT_SETTINGS } from './settings';
-export { AutocompleteInfo, getAutocompleteInfo, setAutocompleteInfo } from './autocomplete';
+export {
+  AutocompleteInfo,
+  getAutocompleteInfo,
+  setAutocompleteInfo,
+  ENTITIES,
+} from './autocomplete';

src/plugins/console/server/lib/proxy_request.ts

@@ -12,6 +12,7 @@ import net from 'net';
 import stream from 'stream';
 import Boom from '@hapi/boom';
 import { URL } from 'url';
+import { sanitizeHostname } from './utils';
 
 interface Args {
   method: 'get' | 'post' | 'put' | 'delete' | 'patch' | 'head';
@@ -23,13 +24,6 @@ interface Args {
   rejectUnauthorized?: boolean;
 }
 
-/**
- * Node http request library does not expect there to be trailing "[" or "]"
- * characters in ipv6 host names.
- */
-const sanitizeHostname = (hostName: string): string =>
-  hostName.trim().replace(/^\[/, '').replace(/\]$/, '');
-
 // We use a modified version of Hapi's Wreck because Hapi, Axios, and Superagent don't support GET requests
 // with bodies, but ES APIs do. Similarly with DELETE requests with bodies. Another library, `request`
 // diverged too much from current behaviour.