[Wheel] Migrate HTTP metadata service to unified package

.github/CODEOWNERS

@@ -15,12 +15,14 @@
 /mooncake-reshard @ShangmingCai @stmatengss @Bo-Vincent @zxpdemonio
 /mooncake-store @ykwd @stmatengss @XucSh @YiXR
 /mooncake-store/*/ha/ @Libotry @YiXR @00fish0 @Icedcoco @Aionw
+/python/mooncake/http_metadata_server.py @ykwd @stmatengss @zxpdemonio
 /python/mooncake/buffer_pool.py @ykwd @stmatengss
 /python/mooncake/ep.py @UNIDY2002 @ympcMark @yuechen-sys
 /python/mooncake/mooncake_config.py @ykwd @stmatengss
 /python/mooncake/mooncake_elastic_buffer.py @UNIDY2002 @ympcMark @yuechen-sys
 /python/mooncake/mooncake_ep_buffer.py @UNIDY2002 @ympcMark @yuechen-sys
 /python/tests/ep/ @UNIDY2002 @ympcMark @yuechen-sys
+/python/tests/services/test_http_metadata_server.py @ykwd @stmatengss @zxpdemonio
 /python/mooncake/cli.py @ykwd @stmatengss @zxpdemonio
 /python/mooncake/cli_client.py @ykwd @stmatengss @zxpdemonio
 /python/mooncake/cli_bench.py @ShangmingCai @alogfans @stmatengss

.github/labeler.yml

@@ -7,6 +7,8 @@ store:
   - changed-files:
     - any-glob-to-any-file:
       - 'mooncake-store/**/*'
+      - 'python/mooncake/http_metadata_server.py'
+      - 'python/tests/services/test_http_metadata_server.py'
       - 'python/mooncake/cli.py'
       - 'python/mooncake/cli_client.py'
       - 'python/mooncake/buffer_pool.py'
@@ -45,6 +47,7 @@ Installation:
   - changed-files:
     - any-glob-to-any-file:
       - 'mooncake-wheel/**/*'
+      - 'python/mooncake/http_metadata_server.py'
       - 'python/mooncake/cli.py'
       - 'python/mooncake/cli_client.py'
       - 'python/mooncake/cli_bench.py'

mooncake-integration/CMakeLists.txt

@@ -219,7 +219,7 @@ endif()
 if(NOT SKBUILD)
   install(
     FILES
-      "${CMAKE_CURRENT_SOURCE_DIR}/../mooncake-wheel/mooncake/http_metadata_server.py"
+      "${CMAKE_CURRENT_SOURCE_DIR}/../python/mooncake/http_metadata_server.py"
       "${CMAKE_CURRENT_SOURCE_DIR}/../python/mooncake/_launcher.py"
       "${CMAKE_CURRENT_SOURCE_DIR}/../python/mooncake/cli_bench.py"
       "${CMAKE_CURRENT_SOURCE_DIR}/../python/mooncake/transfer_engine_topology_dump.py"

mooncake-store/tests/e2e/readme.md

@@ -93,7 +93,7 @@ The script first verifies steady-state `put/get` success with `memory + nof` rep
   - `mooncake-store/src/mooncake_master`
   - `mooncake-integration/store*.so`
 - SPDK has already been built under `extern/spdk`
-- Python environment contains `aiohttp` because the script launches a standalone metadata process with `mooncake-wheel/mooncake/http_metadata_server.py`
+- Python environment contains `aiohttp` because the script launches a standalone metadata process with `python/mooncake/http_metadata_server.py`
 - The script uses `sudo -n` to set hugepages and mount `/dev/hugepages`, so the current user must have passwordless sudo
 
 **Usage**:
@@ -118,7 +118,7 @@ PRE_FAULT_SUCCESS_TARGET=10 BUILD_DIR=/path/to/build ./run_nof_heartbeat_tcp_e2e
 **Notes**:
 
 - The client payload size defaults to `4096` bytes because the current NoF path requires 4K-aligned I/O.
-- The script uses a standalone metadata server process (`mooncake-wheel/mooncake/http_metadata_server.py`) instead of the embedded master metadata server so all four components remain explicit during the test.
+- The script uses a standalone metadata server process (`python/mooncake/http_metadata_server.py`) instead of the embedded master metadata server so all four components remain explicit during the test.
 - In default mode, the script verifies **service continuity** after NoF unmount by checking that post-fault I/O still succeeds.
 - In `CLIENT_GLOBAL_SEGMENT_SIZE=0` mode, the script verifies **NoF-only failure behavior** by checking that post-fault I/O starts failing after the NoF segment is removed.
 - Logs are written under `LOG_DIR` (default `/tmp/mooncake_nof_heartbeat_e2e`) and the final pass/fail summary is printed from `summary.log`.

mooncake-store/tests/e2e/run_nof_heartbeat_tcp_e2e.sh

@@ -90,7 +90,7 @@ python3 "$REPO_ROOT/extern/spdk/scripts/rpc.py" nvmf_create_subsystem "$TARGET_N
 python3 "$REPO_ROOT/extern/spdk/scripts/rpc.py" nvmf_subsystem_add_ns "$TARGET_NQN" Malloc0 >/dev/null || true
 python3 "$REPO_ROOT/extern/spdk/scripts/rpc.py" nvmf_subsystem_add_listener "$TARGET_NQN" -t tcp -a "$TARGET_HOST" -s "$TARGET_PORT" >/dev/null || true
 
-python3 "$REPO_ROOT/mooncake-wheel/mooncake/http_metadata_server.py" --host "$METADATA_HOST" --port "$METADATA_PORT" >"$LOG_DIR/metadata.log" 2>&1 &
+python3 "$REPO_ROOT/python/mooncake/http_metadata_server.py" --host "$METADATA_HOST" --port "$METADATA_PORT" >"$LOG_DIR/metadata.log" 2>&1 &
 META_PID=$!
 sleep 2
 

mooncake-store/tests/e2e/run_oplog_batch_cluster.sh

@@ -233,7 +233,7 @@ require_executable() {
 up_cluster() {
   MASTER_BIN="$BUILD_DIR/mooncake-store/src/mooncake_master"
   INSPECTOR_BIN="$BUILD_DIR/mooncake-store/tools/oplog_batch_inspector"
-  METADATA_SCRIPT="$REPO_ROOT/mooncake-wheel/mooncake/http_metadata_server.py"
+  METADATA_SCRIPT="$REPO_ROOT/python/mooncake/http_metadata_server.py"
   require_executable "$MASTER_BIN"
   [[ "$USE_ETCD_OBSERVER" != true ]] || require_executable "$INSPECTOR_BIN"
   [[ -f "$METADATA_SCRIPT" ]] || die "missing executable: $METADATA_SCRIPT"

mooncake-wheel/tests/test_meta_server.py

@@ -1,4 +1,6 @@
-import os, sys, random
+import os
+import sys
+import random
 import time
 import threading
 
@@ -7,16 +9,19 @@
 # how to test: start mooncake_master and http_metadata_server.
 # Default is tcp, if you want to use rdma, should set MC_RPC_PROTOCOL=rdma and DEVICE_NAME=rdma_xxx at first.
 # ./mooncake_master
-# python mooncake-wheel/mooncake/http_metadata_server.py --port 8080
+# mooncake_http_metadata_server --port 8080
 # python mooncake-wheel/tests/test_meta_server.py 127.0.0.1 8
 
 import mooncake
+
 print(mooncake.__file__)
 
+
 def test_worker(store, num_req):
-  for i in range(num_req):
-    store.is_exist(str(random.randint(0, 1000000)))
-
+    for i in range(num_req):
+        store.is_exist(str(random.randint(0, 1000000)))
+
+
 master_host = "master.mooncake.dc" if len(sys.argv) < 2 else sys.argv[1]
 num_thread = 1 if len(sys.argv) < 3 else int(sys.argv[2])
 # Initialize the store
@@ -30,25 +35,27 @@ def test_worker(store, num_req):
 local_buffer_size = 512 * 1024 * 1024
 master_server_address = os.getenv("MASTER_SERVER", f"{master_host}:50051")
 value_length = 1 * 1024 * 1024
-retcode = store.setup(local_hostname,
-                      metadata_server,
-                      global_segment_size,
-                      local_buffer_size,
-                      protocol,
-                      device_name,
-                      master_server_address)
+retcode = store.setup(
+    local_hostname,
+    metadata_server,
+    global_segment_size,
+    local_buffer_size,
+    protocol,
+    device_name,
+    master_server_address,
+)
 if retcode:
     exit(1)
 time.sleep(1)  # Give some time for initialization
 start_time = time.perf_counter()
 threads = []
 for _ in range(num_thread):
-  thread = threading.Thread(target=test_worker, args=(store, 15000))
-  thread.start()
-  threads.append(thread)
+    thread = threading.Thread(target=test_worker, args=(store, 15000))
+    thread.start()
+    threads.append(thread)
 for thread in threads:
-  thread.join()
+    thread.join()
 end_time = time.perf_counter()
 total_time = end_time - start_time if end_time > start_time else 0
 ops_per_second = 15000 * num_thread / total_time if total_time > 0 else 0
-print(f"total time: {total_time} s, QPS: {ops_per_second:.2f}")
+print(f"total time: {total_time} s, QPS: {ops_per_second:.2f}")

mooncake-wheel/mooncake/http_metadata_server.py → python/mooncake/http_metadata_server.py

@@ -20,6 +20,7 @@
 
 class KVPoll(Enum):
     """Status of the KV server."""
+
     Failed = 0
     Bootstrapping = 1
     WaitingForInput = 2
@@ -29,10 +30,10 @@ class KVPoll(Enum):
 
 class KVBootstrapServer:
     """HTTP server for storing and retrieving metadata."""
-    
+
     def __init__(self, port: int, host: str = "0.0.0.0"):
         """Initialize the server.
-        
+
         Args:
             port: Port to listen on
             host: Host to bind to (default: 0.0.0.0)
@@ -46,7 +47,7 @@ def __init__(self, port: int, host: str = "0.0.0.0"):
         self._runner = None
         self.thread = None
         self._setup_routes()
-    
+
     def run(self):
         """Start the server in a background thread."""
         self.thread = threading.Thread(target=self._run_server, daemon=True)
@@ -56,65 +57,77 @@ def run(self):
 
     def _setup_routes(self):
         """Set up the HTTP routes."""
-        self.app.router.add_route('*', '/metadata', self._handle_metadata)
+        self.app.router.add_route("*", "/metadata", self._handle_metadata)
 
     async def _handle_metadata(self, request: web.Request):
         """Handle metadata requests."""
-        key = request.query.get('key', '').strip()
+        key = request.query.get("key", "").strip()
         if not key:
-            return web.Response(text='metadata key is required', status=400,
-                              content_type='application/json')
-
-        if request.method == 'GET':
+            return web.Response(
+                text="metadata key is required",
+                status=400,
+                content_type="application/json",
+            )
+
+        if request.method == "GET":
             return await self._handle_get(key)
-        elif request.method == 'PUT':
+        elif request.method == "PUT":
             return await self._handle_put(key, request)
-        elif request.method == 'DELETE':
+        elif request.method == "DELETE":
             return await self._handle_delete(key)
-        return web.Response(text='Method not allowed', status=405,
-                          content_type='application/json')
+        return web.Response(
+            text="Method not allowed", status=405, content_type="application/json"
+        )
 
     async def _handle_get(self, key):
         """Handle GET requests."""
         async with self.lock:
             value = self.store.get(key)
         if value is None:
-            return web.Response(text='metadata not found', status=404,
-                              content_type='application/json')
-        return web.Response(body=value, status=200,
-                          content_type='application/json')
+            return web.Response(
+                text="metadata not found", status=404, content_type="application/json"
+            )
+        return web.Response(body=value, status=200, content_type="application/json")
 
     async def _handle_put(self, key, request):
         """Handle PUT requests."""
         data = await request.read()
         async with self.lock:
             if key.find("rpc_meta") != -1 and key in self.store:
-                return web.Response(text='Duplicate rpc_meta key not allowed', status=400,
-                                  content_type='application/json')
+                return web.Response(
+                    text="Duplicate rpc_meta key not allowed",
+                    status=400,
+                    content_type="application/json",
+                )
             self.store[key] = data
-        return web.Response(text='metadata updated', status=200,
-                          content_type='application/json')
+        return web.Response(
+            text="metadata updated", status=200, content_type="application/json"
+        )
 
     async def _handle_delete(self, key):
         """Handle DELETE requests."""
         async with self.lock:
             if key not in self.store:
-                return web.Response(text='metadata not found', status=404,
-                                  content_type='application/json')
+                return web.Response(
+                    text="metadata not found",
+                    status=404,
+                    content_type="application/json",
+                )
             del self.store[key]
-        return web.Response(text='metadata deleted', status=200,
-                          content_type='application/json')
-
+        return web.Response(
+            text="metadata deleted", status=200, content_type="application/json"
+        )
+
     def _run_server(self):
         """Run the server in the current thread."""
         try:
             # Event Loop
             self._loop = asyncio.new_event_loop()
             asyncio.set_event_loop(self._loop)
-            
+
             self._runner = web.AppRunner(self.app)
             self._loop.run_until_complete(self._runner.setup())
-            
+
             site = web.TCPSite(self._runner, host=self.host, port=self.port)
             self._loop.run_until_complete(site.start())
             self._loop.run_forever()
@@ -132,11 +145,11 @@ def close(self):
         if self._loop is not None and self._loop.is_running():
             self._loop.call_soon_threadsafe(self._loop.stop)
             logging.info("Stopping server loop...")
-        
+
         if self.thread is not None and self.thread.is_alive():
             self.thread.join(timeout=2)
             logging.info("Server thread stopped")
-            
+
     def poll(self) -> KVPoll:
         """Poll the server status."""
         if self.thread is None:
@@ -150,60 +163,50 @@ def parse_args():
     """Parse command line arguments."""
     parser = argparse.ArgumentParser(
         description="HTTP Metadata Server for Mooncake",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument("--port", type=int, default=8080, help="Port to listen on")
+    parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind to")
     parser.add_argument(
-        "--port", 
-        type=int, 
-        default=8080, 
-        help="Port to listen on"
-    )
-    parser.add_argument(
-        "--host", 
-        type=str, 
-        default="0.0.0.0", 
-        help="Host to bind to"
-    )
-    parser.add_argument(
-        "--log-level", 
-        type=str, 
+        "--log-level",
+        type=str,
         choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
-        default="INFO", 
-        help="Logging level"
+        default="INFO",
+        help="Logging level",
     )
     return parser.parse_args()
 
 
 def main():
     """Main entry point for the mooncake_http_metadata_server command."""
     args = parse_args()
-    
+
     # Configure logging
     logging.basicConfig(
         level=getattr(logging, args.log_level),
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     )
-    
+
     # Create and start the server
     server = KVBootstrapServer(port=args.port, host=args.host)
     server.run()
-    
+
     # Handle graceful shutdown
     def signal_handler(sig, frame):
         logging.info("Shutting down...")
         server.close()
         sys.exit(0)
-    
+
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
-    
+
     # Keep the main thread alive
     try:
         while True:
             sleep(1)
     except KeyboardInterrupt:
         server.close()
-    
+
     return 0