httpserver: add read-only monitoring module

This patch adds new module httpserver-monitoring-api that provides read-only subset of the API intended for monitoring purposes only. The monitoring flavor of httpserver provides following routes: - /api - /env - /file - /fs - /hardware - /network - /os where each of the routes expose GET-only paths. The monitoring module disables YAML file based configuration as well as SSL connectivity. It is also possible to add trace plugin - libhttpserver-api_trace.so. Please note that monitoring module re-uses the same httpserver-api source code however when compiling we pass '-DMONITORING' flag to disable subset of code not needed for monitoring functionality. We also compile the code with '-fvisibility=hidden' to further reduce size of the monitoring executable (<800K). Fixes #820 Signed-off-by: Waldemar Kozaczuk <[email protected]>
cloudius-systems · Mar 27, 2020 · aa32614 · aa32614
1 parent 9f3bdda
commit aa32614
Show file tree

Hide file tree

Showing 11 changed files with 1,123 additions and 0 deletions.
diff --git a/modules/httpserver-monitoring-api/.gitignore b/modules/httpserver-monitoring-api/.gitignore
@@ -0,0 +1,4 @@
+*.so
+autogen/*
+obj/*
+*.manifest
diff --git a/modules/httpserver-monitoring-api/Makefile b/modules/httpserver-monitoring-api/Makefile
@@ -0,0 +1,84 @@
+SRC = $(shell readlink -f ../..)
+include $(SRC)/modules/java-base/common.gmk
+INCLUDES += -I. -I../httpserver-api -I $(SRC)/libc/internal
+
+# compiler flags:
+#  -g    adds debugging information to the executable file
+#  -Wall turns on most, but not all, compiler warnings
+autodepend = -MD -MT $@ -MP
+CXXFLAGS  = -g -Wall -std=c++11 -fPIC $(INCLUDES) -O2 $(autodepend) -DMONITORING -fvisibility=hidden
+#TODO: Figure out why lto breaks exception handling
+#CXXFLAGS  = -g -Wall -std=c++11 -fPIC $(INCLUDES) -O2 $(autodepend) -DMONITORING -fvisibility=hidden -flto
+#LDFLAGS = -flto
+src = $(shell readlink -f ../..)
+
+ifndef ARCH
+	ARCH = x64
+endif
+
+ifndef mode
+	mode = release
+endif
+
+ifndef OSV_BUILD_PATH
+	OSV_BUILD_PATH = $(src)/build/$(mode).$(ARCH)
+endif
+boost-libs := -lboost_system -lboost_filesystem
+
+# the build target executable:
+TARGET = httpserver-api
+JSON_FILES := $(wildcard api-doc/listings/*.json)
+JSON_CC_FILES := $(subst .json,.json.cc,$(subst api-doc/listings/,autogen/,$(JSON_FILES)))
+CPP_FILES := $(addprefix json/,$(notdir $(wildcard ../httpserver-api/json/*.cc))) \
+	api/fs.cc api/os.cc api/network.cc api/hardware.cc api/env.cc api/file.cc api/api.cc \
+	common.cc main.cc plain_server.cc server.cc connection.cc matcher.cc \
+	reply.cc connection_manager.cc mime_types.cc request_handler.cc \
+	transformers.cc global_server.cc request_parser.cc handlers.cc \
+	path_holder.cc routes.cc $(JSON_CC_FILES)
+OBJ_FILES := $(addprefix obj/,$(CPP_FILES:.cc=.o))
+
+DYN_LIBS = -lpthread -ldl -L../libtools -ltools $(boost-libs)
+
+LIBS = $(DYN_LIBS) $(STATIC_LIBS)
+
+quiet = $(if $V, $1, @echo " $2"; $1)
+very-quiet = $(if $V, $1, @$1)
+
+DEPS := $(OBJ_FILES:.o=.d)
+
+module: all
+
+all: lib$(TARGET).so
+	$(call very-quiet, $(SRC)/scripts/manifest_from_host.sh lib$(TARGET).so > usr.manifest)
+
+lib$(TARGET).so: $(OBJ_FILES)
+	$(call quiet, $(CXX) $(CXXFLAGS) -shared $(STATIC_LIBS) -o $@ $^ $(DYN_LIBS), LINK $@)
+
+ifneq ($(MAKECMDGOALS),clean)
+-include $(DEPS)
+endif
+
+autogen/%.cc: api-doc/listings/% ../httpserver-api/json2code.py
+	$(call very-quiet, mkdir -p autogen)
+	$(call quiet,../httpserver-api/json2code.py -outdir autogen -f $< -ns json, GEN $@)
+
+$(OBJ_FILES): obj/%.o: ../httpserver-api/%.cc
+	$(call very-quiet, mkdir -p obj/stub obj/json obj/api obj/autogen)
+	$(call quiet, $(CXX) $(CXXFLAGS) -c -MMD  -o $@ $<, CXX $@)
+
+clean:
+	$(call quiet, $(RM) -f $(TARGET), CLEAN)
+	$(call very-quiet, $(RM) -f lib*.so)
+	$(call very-quiet, $(RM) -rf obj)
+	$(call very-quiet, $(RM) -rf autogen)
+	$(call very-quiet, $(RM) -f *usr*.manifest)
+
+check:
+	# Test plain readonly HTTP
+	cd $(src) && \
+	make image=httpserver-monitoring-api.fg && \
+	PYTHONPATH=$(src)/scripts modules/httpserver-api/tests/testhttpserver-monitoring-api.py
+
+.PHONY: check
+
+.SECONDARY:
diff --git a/modules/httpserver-monitoring-api/api-doc/listings/api.json b/modules/httpserver-monitoring-api/api-doc/listings/api.json
@@ -0,0 +1,40 @@
+{
+   "apiVersion":"0.0.1",
+   "swaggerVersion":"1.2",
+   "basePath":"{{Protocol}}://{{Host}}",
+   "resourcePath":"/api",
+   "produces":[
+      "application/json"
+   ],
+   "apis":[
+      {
+         "path":"/api/batch",
+         "operations":[
+            {
+               "method":"POST",
+               "summary":"Batch process API calls",
+               "notes":"Perform batch API calls in a single command. Commands are performed sequentially and independently. Each command has its own response code",
+               "type":"string",
+               "errorResponses":[
+                  {
+                     "code":400,
+                     "reason":"Bad request"
+                  }
+               ],
+               "nickname":"api_batch",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                  "name":"batch",
+                  "paramType":"form",
+                  "type":"string",
+                  "required":true
+                  }
+               ]
+            }
+         ]
+      }
+   ]
+}
diff --git a/modules/httpserver-monitoring-api/api-doc/listings/env.json b/modules/httpserver-monitoring-api/api-doc/listings/env.json
@@ -0,0 +1,57 @@
+{
+   "apiVersion":"0.0.1",
+   "swaggerVersion":"1.2",
+   "basePath":"{{Protocol}}://{{Host}}",
+   "resourcePath":"/env",
+   "produces":[
+      "application/json"
+   ],
+   "apis":[
+      {
+         "path":"/env/{var}",
+         "operations":[
+            {
+               "method":"GET",
+               "summary":"Get an environment variable",
+               "notes":"return the environment variable value",
+               "type":"string",
+               "errorResponses":[
+                  {
+                     "code":400,
+                     "reason":"Variable not found"
+                  }
+               ],
+               "nickname":"getEnv",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"var",
+                     "description":"name of the environment variable",
+                     "required":true,
+                     "allowMultiple":true,
+                     "type":"string",
+                     "paramType":"path"
+                  }
+               ]
+            }
+         ]
+      },
+            {
+         "path":"/env/",
+         "operations":[
+            {
+               "method":"GET",
+               "summary":"Returns a list of all environment variables in the system.",
+               "type":"array",
+               "items": {"type": "string"},
+               "nickname":"list_env",
+               "produces":[
+                  "application/json"
+               ]
+               }
+               ]
+            }
+         ]
+}
diff --git a/modules/httpserver-monitoring-api/api-doc/listings/file.json b/modules/httpserver-monitoring-api/api-doc/listings/file.json
@@ -0,0 +1,187 @@
+{
+   "apiVersion":"0.0.1",
+   "swaggerVersion":"1.2",
+   "basePath":"{{Protocol}}://{{Host}}",
+   "resourcePath":"/file",
+   "produces":[
+      "application/json"
+   ],
+   "apis":[
+      {
+         "path":"/file/{path-par}",
+         "operations":[
+            {
+               "method":"GET",
+               "summary":"Get File/Directory information",
+               "notes":"return File or Directory related information",
+               "type":"string",
+               "errorResponses":[
+                  {
+                     "code":404,
+                     "reason":"File not found"
+                  },
+                  {
+                     "code":400,
+                     "reason":"Bad Request"
+                  }
+               ],
+               "nickname":"getFile",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"path-par",
+                     "description":"Full path of file or directory",
+                     "required":true,
+                     "allowMultiple":true,
+                     "type":"string",
+                     "paramType":"path"
+                  },
+                  {
+                     "name":"op",
+                     "description":"The operation to perform",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query",
+                     "enum":["GET", "LISTSTATUS", "GETFILESTATUS"]
+                  },
+                  {
+                     "name":"offset",
+                     "description":"Offset in a file",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"long",
+                     "paramType":"query"
+                  },
+                  {
+                     "name":"length",
+                     "description":"The number of bytes to be processed.",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"long",
+                     "paramType":"query"
+                  }
+               ]
+            }
+         ]
+      }
+   ],
+   "models":{
+      "ContentSummary":{
+         "id": "ContentSummary",
+         "properties":{
+            "directoryCount":{
+               "description":"The number of directories.",
+               "type":"int",
+               "required":true
+            },
+            "fileCount":{
+               "description":"The number of files.",
+               "type":"int",
+               "required":true
+            },
+            "length":{
+               "description":"The number of bytes used by the content.",
+               "type":"int",
+               "required":true
+            },
+            "quota":{
+               "description":"The namespace quota of this directory.",
+               "type":"int",
+               "required":true
+            },
+            "spaceConsumed":{
+               "description":"The disk space consumed by the content.",
+               "type":"int",
+               "required":true
+            },
+            "spaceQuota":{
+               "description":"The disk space quota.",
+               "type":"int",
+               "required":true
+            }
+         }
+      },
+      "FileChecksum":{
+         "id": "FileChecksum",
+         "properties":{
+            "algorithm":{
+               "description":"The name of the checksum algorithm.",
+               "type":"string",
+               "required":true
+            },
+            "bytes":{
+               "description":"The byte sequence of the checksum in hexadecimal.",
+               "type":"string",
+               "required":true
+            },
+            "length":{
+               "description":"The length of the bytes (not the length of the string).",
+               "type":"int",
+               "required":true
+            }
+         }
+      },
+      "FileStatusProperties":{
+         "id": "FileStatusProperties",
+         "properties":{
+            "accessTime":{
+               "description":"The access time.",
+               "type":"int",
+               "required":true
+            },
+            "blockSize":{
+               "description":"The block size of a file.",
+               "type":"int",
+               "required":true
+            },
+            "group":{
+               "description":"The group owner.",
+               "type":"string",
+               "required":true
+            },
+            "length":{
+               "description":"The number of bytes in a file.",
+               "type":"long",
+               "required":true
+            },
+            "modificationTime":{
+               "description":"The modification time.",
+               "type":"int",
+               "required":true
+            },
+            "owner":{
+               "description":"The user who is the owner.",
+               "type":"string",
+               "required":true
+            },
+            "pathSuffix":{
+               "description":"The path suffix.",
+               "type":"string",
+               "required":true
+            },
+            "permission":{
+               "description":"The permission represented as a octal string.",
+               "type":"string",
+               "required":true
+            },
+            "replication":{
+               "description":"The number of replication of a file.",
+               "type":"int",
+               "required":true
+            },
+            "symlink":{
+               "description":"The link target of a symlink.",
+               "type":"string"
+            },
+            "type":{
+               "description":"The type of the path object.",
+               "type":"string",
+               "required":true
+            }
+         }
+      }
+   }
+}