diff --git a/areal/experimental/engine/archon_checkpoint.py b/areal/experimental/engine/archon_checkpoint.py
index 58eb3423c5..679171ae4d 100644
--- a/areal/experimental/engine/archon_checkpoint.py
+++ b/areal/experimental/engine/archon_checkpoint.py
@@ -121,6 +121,31 @@ def load_model_from_hf(engine: ArchonEngine, path: str) -> None:
     # Convert back to Archon format
     archon_state_dict = engine.state_dict_adapter.from_hf(hf_state_dict)
 
+    # Compute key differences for diagnostics
+    model_keys = set(state_dict.keys())
+    loaded_keys = set(archon_state_dict.keys())
+
+    missing_keys = model_keys - loaded_keys
+    unexpected_keys = loaded_keys - model_keys
+
+    # Filter known expected missing keys
+    expected_missing = set()
+    for key in list(missing_keys):
+        # rotary_emb is computed at runtime, not stored in checkpoint
+        if "rotary_emb" in key:
+            expected_missing.add(key)
+    missing_keys -= expected_missing
+
+    if dist.get_rank() == 0:
+        if missing_keys:
+            engine.logger.warning(
+                f"Unexpected missing keys in checkpoint: {missing_keys}"
+            )
+        if unexpected_keys:
+            engine.logger.warning(
+                f"Unexpected extra keys in checkpoint: {unexpected_keys}"
+            )
+
     # Load into FSDP model (same as DCPState.load_state_dict)
     set_model_state_dict(
         engine.model,
@@ -142,6 +167,8 @@ def save_to_dcp(engine: ArchonEngine, path: str, with_optim: bool) -> None:
     state_dict = {"dcp": dcp_state}
     dcp.save(state_dict, checkpoint_id=path)
 
+    dist.barrier(group=engine.cpu_group)
+
 
 def load_from_dcp(engine: ArchonEngine, path: str, with_optim: bool) -> None:
     """Load model (and optionally optimizer) from DCP format."""
@@ -152,6 +179,8 @@ def load_from_dcp(engine: ArchonEngine, path: str, with_optim: bool) -> None:
     state_dict = {"dcp": dcp_state}
     dcp.load(state_dict=state_dict, checkpoint_id=path)
 
+    dist.barrier(group=engine.cpu_group)
+
 
 def save_optimizer_state(engine: ArchonEngine, path: str) -> None:
     """Save optimizer state to disk (sharded by rank)."""
diff --git a/areal/experimental/engine/archon_engine.py b/areal/experimental/engine/archon_engine.py
index 17c8669f18..3858b6a09a 100644
--- a/areal/experimental/engine/archon_engine.py
+++ b/areal/experimental/engine/archon_engine.py
@@ -85,7 +85,6 @@
 from areal.utils.network import find_free_ports, gethostip
 from areal.utils.offload import torch_memory_saver
 from areal.utils.perf_tracer import trace_perf
-from areal.utils.save_load import get_state_dict_from_repo_id_or_path
 
 
 @dataclass
@@ -265,6 +264,8 @@ def initialize(self, addr: str | None, ft_spec: FinetuneSpec, *args, **kwargs):
             f"Applied parallelism in {time.perf_counter() - tik:.2f} seconds"
         )
 
+        self._materialize_and_load_weights()
+
         self._create_optimizer(ft_spec)
 
         self._initialized = True
@@ -817,12 +818,20 @@ def _create_device_model(self):
         self.tokenizer = load_hf_tokenizer(self.config.path)
 
         tik = time.perf_counter()
-        with torch.device(current_platform.device_type):
-            model = self._create_llm_model()
 
-        self.logger.info(f"Model creation time: {time.perf_counter() - tik:.2f}s")
+        # Meta device mode: create structure only, no memory allocation
+        # Parameters exist only as metadata until materialized after FSDP
+        with torch.device("meta"):
+            model = self._create_model_structure()
+        model = model.to(getattr(torch, self.config.dtype))
         self.model = model
 
+        self.logger.info(
+            f"Model structure created on meta device in "
+            f"{time.perf_counter() - tik:.2f}s"
+        )
+        self.get_device_stats().log("after create model structure")
+
     def _build_ac_config(self) -> ActivationCheckpointConfig | None:
         # First check if gradient checkpointing is enabled
         if not self.config.gradient_checkpointing:
@@ -851,56 +860,43 @@ def _build_ac_config(self) -> ActivationCheckpointConfig | None:
 
         return ac_config
 
-    def _create_llm_model(self) -> nn.Module:
+    def _create_model_structure(self) -> nn.Module:
+        """Create model structure on meta device without loading weights."""
         model_args = self.spec.model_args_class.from_hf_config(
             self.model_config,
             is_critic=self.config.is_critic,
             attn_type=self.config.archon.attn_type,
         )
         model = self.spec.model_class(model_args)
+        return model
 
-        if not self.config.init_from_scratch:
-            self._load_hf_weights_to_archon_model(model)
+    def _materialize_and_load_weights(self):
+        """Materialize meta tensors and load weights after FSDP parallelization."""
+        if self.config.archon.offload_params:
+            init_device = "cpu"
+            buffer_device = current_platform.device_type
         else:
-            model.init_weights()
+            init_device = current_platform.device_type
+            buffer_device = init_device
 
-        dtype = getattr(torch, self.config.dtype)
-        model = model.to(dtype)
-
-        return model
+        tik = time.perf_counter()
 
-    def _load_hf_weights_to_archon_model(self, model: nn.Module):
-        self.logger.info(f"Loading weights from {self.config.path}")
+        self.model.to_empty(device=init_device)
 
-        hf_state_dict = get_state_dict_from_repo_id_or_path(self.config.path)
+        if not self.config.init_from_scratch:
+            load_model_from_hf(self, self.config.path)
+        else:
+            with torch.no_grad():
+                self.model.init_weights()
 
-        adapter = self.spec.state_dict_adapter_class(self.model_config)
+        self.model.init_buffers(buffer_device=buffer_device)
 
-        archon_state_dict = adapter.from_hf(hf_state_dict)
+        dist.barrier(group=self.cpu_group)
 
-        missing_keys, unexpected_keys = model.load_state_dict(
-            archon_state_dict, strict=False
+        self.logger.info(
+            f"Materialized and loaded weights in {time.perf_counter() - tik:.2f}s"
         )
-
-        if missing_keys:
-            expected_missing = {"rope_cache"}
-            if not self.config.is_critic:
-                expected_missing.add("score.weight")
-            else:
-                expected_missing.add("output.weight")
-
-            actual_missing = [k for k in missing_keys if k not in expected_missing]
-            if actual_missing:
-                self.logger.warning(
-                    f"Missing keys when loading weights: {actual_missing}"
-                )
-
-        if unexpected_keys:
-            self.logger.warning(
-                f"Unexpected keys when loading weights: {unexpected_keys}"
-            )
-
-        self.logger.info("Successfully loaded weights into Archon model")
+        self.get_device_stats().log("after materialize and load weights")
 
     def _create_optimizer(self, ft_spec: FinetuneSpec):
         if self.optimizer_config is None:
diff --git a/areal/experimental/models/archon/base.py b/areal/experimental/models/archon/base.py
index 43daf98ff2..1a3a32935a 100644
--- a/areal/experimental/models/archon/base.py
+++ b/areal/experimental/models/archon/base.py
@@ -117,13 +117,6 @@ def convert_single_to_hf(
 class BaseArchonModel(nn.Module, ABC):
     """Base class for Archon models."""
 
-    model_args: BaseModelArgs
-    layers: nn.ModuleDict
-    tok_embeddings: nn.Embedding | None
-    norm: nn.Module | None
-    output: nn.Linear | None
-    score: nn.Linear | None
-
     @abstractmethod
     def forward(
         self,
@@ -134,7 +127,14 @@ def forward(
     ) -> torch.Tensor: ...
 
     @abstractmethod
-    def init_weights(self, buffer_device: torch.device | None = None) -> None: ...
+    def init_weights(self) -> None:
+        """Initialize model parameters."""
+        ...
+
+    @abstractmethod
+    def init_buffers(self, buffer_device: torch.device | str) -> None:
+        """Initialize model buffers (e.g., rope_cache)."""
+        ...
 
 
 __all__ = [
diff --git a/areal/experimental/models/archon/moe/moe.py b/areal/experimental/models/archon/moe/moe.py
index da58bc88cd..707cc5b441 100644
--- a/areal/experimental/models/archon/moe/moe.py
+++ b/areal/experimental/models/archon/moe/moe.py
@@ -104,21 +104,8 @@ def __init__(self, moe_args: MoEArgs, dim: int, hidden_dim: int):
         # Buffers for auxiliary-loss-free load balancing
         # expert_bias is used to adjust routing probabilities
         # tokens_per_expert tracks usage for bias updates
-        if self.load_balance_coeff is not None:
-            assert self.load_balance_coeff > 0.0
-            self.register_buffer(
-                "expert_bias",
-                torch.zeros(moe_args.num_experts, dtype=torch.float32),
-                persistent=True,
-            )
-        else:
-            self.expert_bias = None
-
-        self.register_buffer(
-            "tokens_per_expert",
-            torch.zeros(moe_args.num_experts, dtype=torch.float32),
-            persistent=False,
-        )
+        self.expert_bias: torch.Tensor | None
+        self.tokens_per_expert: torch.Tensor
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         """Forward pass through MoE layer.
@@ -214,12 +201,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
 
         return out.view(bs, slen, dim)
 
-    def init_weights(self, init_std: float, buffer_device: torch.device):
-        """Initialize weights.
+    def init_weights(self, init_std: float):
+        """Initialize MoE parameters.
 
         Args:
             init_std: Standard deviation for output projections.
-            buffer_device: Device for buffers.
         """
         self.experts.init_weights(init_std)
         self.router.init_weights(init_std)
@@ -227,11 +213,27 @@ def init_weights(self, init_std: float, buffer_device: torch.device):
         if self.shared_experts is not None:
             self.shared_experts.init_weights(init_std)
 
-        # Reset buffers
+    def init_buffers(self, buffer_device: torch.device | str):
+        """Initialize MoE buffers (tokens_per_expert, expert_bias).
+
+        Args:
+            buffer_device: Device for buffers.
+        """
         with torch.device(buffer_device):
-            self.tokens_per_expert = torch.zeros(self.num_experts, dtype=torch.float32)
+            self.register_buffer(
+                "tokens_per_expert",
+                torch.zeros(self.num_experts, dtype=torch.float32),
+                persistent=False,
+            )
             if self.load_balance_coeff is not None:
-                self.expert_bias = torch.zeros(self.num_experts, dtype=torch.float32)
+                assert self.load_balance_coeff > 0.0
+                self.register_buffer(
+                    "expert_bias",
+                    torch.zeros(self.num_experts, dtype=torch.float32),
+                    persistent=True,
+                )
+            else:
+                self.expert_bias = None
 
 
 __all__ = ["MoE", "FeedForward"]
diff --git a/areal/experimental/models/archon/qwen2/model/model.py b/areal/experimental/models/archon/qwen2/model/model.py
index e936ae4298..11f6175559 100644
--- a/areal/experimental/models/archon/qwen2/model/model.py
+++ b/areal/experimental/models/archon/qwen2/model/model.py
@@ -10,6 +10,7 @@
     SDPAWrapper,
     VarlenAttentionWrapper,
 )
+from areal.experimental.models.archon.base import BaseArchonModel
 from areal.experimental.models.archon.qwen2.model.args import Qwen2ModelArgs
 from areal.experimental.models.archon.qwen2.model.rope import (
     apply_rotary_emb,
@@ -241,14 +242,14 @@ def forward(
         x = x + self.feed_forward(self.ffn_norm(x))
         return x
 
-    def init_weights(self, buffer_device: torch.device):
+    def init_weights(self):
         for norm in (self.attention_norm, self.ffn_norm):
             norm.reset_parameters()
         self.attention.init_weights(self.weight_init_std)
         self.feed_forward.init_weights(self.weight_init_std)
 
 
-class Qwen2Model(nn.Module):
+class Qwen2Model(BaseArchonModel):
     """Qwen2 transformer model."""
 
     def __init__(self, model_args: Qwen2ModelArgs):
@@ -286,17 +287,13 @@ def _precompute_rope_cache(self) -> torch.Tensor:
             self.model_args.rope_theta,
         )
 
-    def init_weights(self, buffer_device: torch.device | None = None):
-        buffer_device = buffer_device or self.rope_cache.device
-        with torch.device(buffer_device):
-            self.rope_cache = self._precompute_rope_cache()
-
+    def init_weights(self):
         if self.tok_embeddings is not None:
             nn.init.normal_(self.tok_embeddings.weight)
 
         for layer in self.layers.values():
             if layer is not None:
-                layer.init_weights(buffer_device)
+                layer.init_weights()
 
         if self.norm is not None:
             self.norm.reset_parameters()
@@ -322,6 +319,10 @@ def init_weights(self, buffer_device: torch.device | None = None):
                 b=cutoff_factor * final_out_std,
             )
 
+    def init_buffers(self, buffer_device: torch.device | str):
+        with torch.device(buffer_device):
+            self.rope_cache = self._precompute_rope_cache()
+
     def forward(
         self,
         tokens: torch.Tensor,
diff --git a/areal/experimental/models/archon/qwen3/model/model.py b/areal/experimental/models/archon/qwen3/model/model.py
index 100dc541be..6031b9520f 100644
--- a/areal/experimental/models/archon/qwen3/model/model.py
+++ b/areal/experimental/models/archon/qwen3/model/model.py
@@ -11,6 +11,7 @@
     SDPAWrapper,
     VarlenAttentionWrapper,
 )
+from areal.experimental.models.archon.base import BaseArchonModel
 from areal.experimental.models.archon.moe import MoE
 from areal.experimental.models.archon.qwen3.model.args import Qwen3ModelArgs
 from areal.experimental.models.archon.qwen3.model.rope import (
@@ -336,17 +337,27 @@ def forward(
             x = x + self.feed_forward(self.ffn_norm(x))
         return x
 
-    def init_weights(self, buffer_device: torch.device):
+    def init_weights(self):
+        """Initialize layer parameters."""
         for norm in (self.attention_norm, self.ffn_norm):
             norm.reset_parameters()
         self.attention.init_weights(self.weight_init_std)
         if self.moe_enabled:
-            self.moe.init_weights(self.weight_init_std, buffer_device)
+            self.moe.init_weights(self.weight_init_std)
         else:
             self.feed_forward.init_weights(self.weight_init_std)
 
+    def init_buffers(self, buffer_device: torch.device | str):
+        """Initialize layer buffers (MoE buffers if enabled).
 
-class Qwen3Model(nn.Module):
+        Args:
+            buffer_device: Device for buffers.
+        """
+        if self.moe_enabled:
+            self.moe.init_buffers(buffer_device)
+
+
+class Qwen3Model(BaseArchonModel):
     """Qwen3 transformer model."""
 
     def __init__(self, model_args: Qwen3ModelArgs):
@@ -384,17 +395,14 @@ def _precompute_rope_cache(self) -> torch.Tensor:
             self.model_args.rope_theta,
         )
 
-    def init_weights(self, buffer_device: torch.device | None = None):
-        buffer_device = buffer_device or self.rope_cache.device
-        with torch.device(buffer_device):
-            self.rope_cache = self._precompute_rope_cache()
-
+    def init_weights(self):
+        """Initialize model parameters."""
         if self.tok_embeddings is not None:
             nn.init.normal_(self.tok_embeddings.weight)
 
         for layer in self.layers.values():
             if layer is not None:
-                layer.init_weights(buffer_device)
+                layer.init_weights()
 
         if self.norm is not None:
             self.norm.reset_parameters()
@@ -420,6 +428,19 @@ def init_weights(self, buffer_device: torch.device | None = None):
                 b=cutoff_factor * final_out_std,
             )
 
+    def init_buffers(self, buffer_device: torch.device | str):
+        """Initialize model buffers (rope_cache and MoE buffers).
+
+        Args:
+            buffer_device: Device for buffers.
+        """
+        with torch.device(buffer_device):
+            self.rope_cache = self._precompute_rope_cache()
+        # Initialize MoE buffers in each layer
+        for layer in self.layers.values():
+            if layer is not None:
+                layer.init_buffers(buffer_device)
+
     def forward(
         self,
         tokens: torch.Tensor,
diff --git a/areal/tests/experimental/archon/test_moe.py b/areal/tests/experimental/archon/test_moe.py
index db8c713943..44495cfd68 100644
--- a/areal/tests/experimental/archon/test_moe.py
+++ b/areal/tests/experimental/archon/test_moe.py
@@ -40,7 +40,8 @@ def test_forward_shape(self):
         moe_args = MoEArgs(num_experts=4, top_k=2, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(batch_size, seq_len, dim, device="cuda")
         out = moe(x)
@@ -56,7 +57,8 @@ def test_forward_values(self):
         moe_args = MoEArgs(num_experts=4, top_k=1, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(batch_size, seq_len, dim, device="cuda")
         out = moe(x)
@@ -77,7 +79,8 @@ def test_top_k_1(self):
         moe_args = MoEArgs(num_experts=4, top_k=1, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
         out = moe(x)
@@ -91,7 +94,8 @@ def test_top_k_2(self):
         moe_args = MoEArgs(num_experts=8, top_k=2, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
         out = moe(x)
@@ -105,7 +109,8 @@ def test_many_experts(self):
         moe_args = MoEArgs(num_experts=64, top_k=8, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
         out = moe(x)
@@ -125,7 +130,8 @@ def test_with_shared_experts(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         assert moe.shared_experts is not None
 
@@ -143,7 +149,8 @@ def test_without_shared_experts(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         assert moe.shared_experts is None
 
@@ -168,7 +175,8 @@ def test_score_before_experts(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
         out = moe(x)
@@ -187,7 +195,8 @@ def test_score_after_experts(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
         out = moe(x)
@@ -207,7 +216,8 @@ def test_tokens_per_expert_tracking(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         # Initially zero
         assert torch.all(moe.tokens_per_expert == 0)
@@ -231,6 +241,7 @@ def test_expert_bias_buffer(self):
             num_experts=4, top_k=2, load_balance_coeff=1e-3, use_grouped_mm=False
         )
         moe = MoE(moe_args, dim, hidden_dim).cuda()
+        moe.init_buffers(buffer_device=torch.device("cuda"))
         assert moe.expert_bias is not None
 
         # Without load balancing
@@ -238,6 +249,7 @@ def test_expert_bias_buffer(self):
             num_experts=4, top_k=2, load_balance_coeff=None, use_grouped_mm=False
         )
         moe = MoE(moe_args, dim, hidden_dim).cuda()
+        moe.init_buffers(buffer_device=torch.device("cuda"))
         assert moe.expert_bias is None
 
 
@@ -251,7 +263,8 @@ def test_gradient_flow(self):
         moe_args = MoEArgs(num_experts=4, top_k=2, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda", requires_grad=True)
         out = moe(x)
@@ -276,7 +289,8 @@ def test_gradient_all_experts(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         # Use enough tokens so each expert gets some
         x = torch.randn(1, num_experts * 4, dim, device="cuda", requires_grad=True)
@@ -307,7 +321,8 @@ def test_qwen3_like_config(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(1, 16, dim, device="cuda")
         out = moe(x)
@@ -323,7 +338,8 @@ def test_multiple_forward_passes(self):
         )
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(2, 8, dim, device="cuda")
 
@@ -343,7 +359,8 @@ def test_batch_size_1(self):
         moe_args = MoEArgs(num_experts=4, top_k=2, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(1, 16, dim, device="cuda")
         out = moe(x)
@@ -357,7 +374,8 @@ def test_single_token(self):
         moe_args = MoEArgs(num_experts=4, top_k=2, use_grouped_mm=False)
 
         moe = MoE(moe_args, dim, hidden_dim).cuda()
-        moe.init_weights(init_std=0.02, buffer_device=torch.device("cuda"))
+        moe.init_weights(init_std=0.02)
+        moe.init_buffers(buffer_device=torch.device("cuda"))
 
         x = torch.randn(1, 1, dim, device="cuda")
         out = moe(x)
diff --git a/areal/tests/experimental/archon/test_qwen3_model_moe.py b/areal/tests/experimental/archon/test_qwen3_model_moe.py
index 4a15c8fcb4..f991b8aeaf 100644
--- a/areal/tests/experimental/archon/test_qwen3_model_moe.py
+++ b/areal/tests/experimental/archon/test_qwen3_model_moe.py
@@ -200,7 +200,8 @@ def test_moe_block_creation(self, moe_args):
     def test_dense_block_forward(self, dense_args):
         """Test forward pass for dense TransformerBlock."""
         block = TransformerBlock(layer_id=0, model_args=dense_args)
-        block.init_weights(torch.device("cpu"))
+        block.init_weights()
+        block.init_buffers(buffer_device=torch.device("cpu"))
 
         bs, seq_len = 2, 8
         x = torch.randn(bs, seq_len, dense_args.dim)
@@ -237,7 +238,8 @@ def test_dense_block_forward(self, dense_args):
     def test_moe_block_forward(self, moe_args):
         """Test forward pass for MoE TransformerBlock."""
         block = TransformerBlock(layer_id=0, model_args=moe_args).cuda()
-        block.init_weights(torch.device("cuda"))
+        block.init_weights()
+        block.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 8
         x = torch.randn(bs, seq_len, moe_args.dim, device="cuda")
@@ -280,7 +282,8 @@ def test_moe_block_forward(self, moe_args):
     def test_moe_block_gradient_flow(self, moe_args):
         """Test gradient flow through MoE TransformerBlock."""
         block = TransformerBlock(layer_id=0, model_args=moe_args).cuda()
-        block.init_weights(torch.device("cuda"))
+        block.init_weights()
+        block.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 8
         x = torch.randn(bs, seq_len, moe_args.dim, device="cuda", requires_grad=True)
@@ -381,6 +384,7 @@ def test_dense_model_forward(self, dense_model_args):
         """Test forward pass for dense model."""
         model = Qwen3Model(dense_model_args)
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cpu"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(0, dense_model_args.vocab_size, (bs, seq_len))
@@ -403,6 +407,7 @@ def test_moe_model_forward(self, moe_model_args):
         """Test forward pass for full MoE model."""
         model = Qwen3Model(moe_model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(
@@ -427,6 +432,7 @@ def test_mixed_model_forward(self, mixed_model_args):
         """Test forward pass for mixed MoE/dense model."""
         model = Qwen3Model(mixed_model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(
@@ -480,6 +486,7 @@ def test_moe_model_gradient_flow(self, moe_model_args):
         """Test gradient flow through MoE model."""
         model = Qwen3Model(moe_model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(
@@ -516,6 +523,7 @@ def test_mixed_model_gradient_flow(self, mixed_model_args):
         """Test gradient flow through mixed MoE/dense model."""
         model = Qwen3Model(mixed_model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(
@@ -550,6 +558,7 @@ def test_moe_model_with_positions(self, moe_model_args):
         """Test MoE model with explicit positions."""
         model = Qwen3Model(moe_model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(
@@ -591,6 +600,7 @@ def test_moe_critic_model(self):
         )
         model = Qwen3Model(model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 16
         tokens = torch.randint(0, model_args.vocab_size, (bs, seq_len), device="cuda")
@@ -640,6 +650,7 @@ def test_qwen3_30b_a3b_like_config(self):
 
         model = Qwen3Model(model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 1, 16
         tokens = torch.randint(0, model_args.vocab_size, (bs, seq_len), device="cuda")
@@ -683,6 +694,7 @@ def test_with_shared_experts(self):
 
         model = Qwen3Model(model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 2, 8
         tokens = torch.randint(0, model_args.vocab_size, (bs, seq_len), device="cuda")
@@ -726,6 +738,7 @@ def test_single_token_inference(self):
 
         model = Qwen3Model(model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
         model.eval()
 
         bs = 1
@@ -766,6 +779,7 @@ def test_batch_size_1(self):
 
         model = Qwen3Model(model_args).cuda()
         model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cuda"))
 
         bs, seq_len = 1, 32
         tokens = torch.randint(0, model_args.vocab_size, (bs, seq_len), device="cuda")
diff --git a/areal/tests/experimental/archon/test_state_dict_adapter.py b/areal/tests/experimental/archon/test_state_dict_adapter.py
index a210da8848..cc80c369da 100644
--- a/areal/tests/experimental/archon/test_state_dict_adapter.py
+++ b/areal/tests/experimental/archon/test_state_dict_adapter.py
@@ -449,7 +449,8 @@ def test_moe_weight_roundtrip_forward_match(
 
         # Create and initialize model
         model1 = Qwen3Model(moe_model_args)
-        model1.init_weights(device)
+        model1.init_weights()
+        model1.init_buffers(buffer_device=device)
         model1.to(device)
 
         # Create adapter
@@ -464,6 +465,7 @@ def test_moe_weight_roundtrip_forward_match(
         # that doesn't exist in HF models (it's initialized to zeros)
         roundtrip_archon_state = adapter.from_hf(hf_state)
         model2 = Qwen3Model(moe_model_args)
+        model2.init_buffers(buffer_device=device)
         model2.load_state_dict(roundtrip_archon_state, strict=False)
         model2.to(device)
 
@@ -493,7 +495,8 @@ def test_moe_weight_keys_preserved(self, moe_model_args, moe_adapter_config):
         from areal.experimental.models.archon.qwen3.model.model import Qwen3Model
 
         model = Qwen3Model(moe_model_args)
-        model.init_weights(torch.device("cpu"))
+        model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cpu"))
 
         archon_state = model.state_dict()
         adapter = Qwen3StateDictAdapter(moe_adapter_config)
@@ -519,7 +522,8 @@ def test_moe_weight_values_preserved(self, moe_model_args, moe_adapter_config):
         from areal.experimental.models.archon.qwen3.model.model import Qwen3Model
 
         model = Qwen3Model(moe_model_args)
-        model.init_weights(torch.device("cpu"))
+        model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cpu"))
 
         archon_state = model.state_dict()
         adapter = Qwen3StateDictAdapter(moe_adapter_config)
@@ -553,7 +557,8 @@ def test_mixed_moe_dense_layers_preserved(self, moe_model_args, moe_adapter_conf
         # - layer 2: dense (FFN)
         # - layer 3: MoE
         model = Qwen3Model(moe_model_args)
-        model.init_weights(torch.device("cpu"))
+        model.init_weights()
+        model.init_buffers(buffer_device=torch.device("cpu"))
 
         # Verify layer structure
         assert model.layers["0"].moe is None
diff --git a/areal/tests/experimental/archon/test_weight_sync.py b/areal/tests/experimental/archon/test_weight_sync.py
index 88f6ae49d2..cb596390c7 100644
--- a/areal/tests/experimental/archon/test_weight_sync.py
+++ b/areal/tests/experimental/archon/test_weight_sync.py
@@ -96,7 +96,8 @@ def small_dense_model(small_dense_model_args):
     from areal.experimental.models.archon.qwen3.model.model import Qwen3Model
 
     model = Qwen3Model(small_dense_model_args)
-    model.init_weights(torch.device("cpu"))
+    model.init_weights()
+    model.init_buffers(buffer_device=torch.device("cpu"))
     return model
 
 
@@ -106,7 +107,8 @@ def small_moe_model(small_moe_model_args):
     from areal.experimental.models.archon.qwen3.model.model import Qwen3Model
 
     model = Qwen3Model(small_moe_model_args)
-    model.init_weights(torch.device("cpu"))
+    model.init_weights()
+    model.init_buffers(buffer_device=torch.device("cpu"))
     return model
 
 
diff --git a/areal/tests/experimental/archon/torchrun/dist_utils.py b/areal/tests/experimental/archon/torchrun/dist_utils.py
index 802a7270bf..8c78fad8cd 100644
--- a/areal/tests/experimental/archon/torchrun/dist_utils.py
+++ b/areal/tests/experimental/archon/torchrun/dist_utils.py
@@ -96,7 +96,8 @@ def create_golden_model(
     """Create non-parallelized model for comparison."""
     torch.manual_seed(seed)
     model = Qwen3Model(model_args)
-    model.init_weights(device)
+    model.init_weights()
+    model.init_buffers(buffer_device=device)
     model = model.to(device)
     return model
 
diff --git a/areal/tests/experimental/archon/torchrun/run_ep_tests.py b/areal/tests/experimental/archon/torchrun/run_ep_tests.py
index bd1168c91e..5b1b0cfc52 100644
--- a/areal/tests/experimental/archon/torchrun/run_ep_tests.py
+++ b/areal/tests/experimental/archon/torchrun/run_ep_tests.py
@@ -131,7 +131,8 @@ def create_ep_model(
     """Create and parallelize model with given parallel dims."""
     torch.manual_seed(seed)
     model = Qwen3Model(model_args)
-    model.init_weights(device)
+    model.init_weights()
+    model.init_buffers(buffer_device=device)
     model = model.to(device)
 
     parallelize_qwen3(
@@ -318,7 +319,8 @@ def test_weight_sync(
 
     torch.manual_seed(42)
     model = Qwen3Model(model_args)
-    model.init_weights(device)
+    model.init_weights()
+    model.init_buffers(buffer_device=device)
     model = model.to(device)
     original_state = {k: v.clone() for k, v in model.state_dict().items()}
 
@@ -375,7 +377,8 @@ def test_weight_sync(
 
     torch.manual_seed(42)
     model_new = Qwen3Model(model_args)
-    model_new.init_weights(device)
+    model_new.init_weights()
+    model_new.init_buffers(buffer_device=device)
     model_new = model_new.to(device)
 
     loadable_state = {}
@@ -453,7 +456,8 @@ def test_state_dict_update(output: str | None = None) -> bool:
 
     torch.manual_seed(99)
     model_new = Qwen3Model(model_args)
-    model_new.init_weights(device)
+    model_new.init_weights()
+    model_new.init_buffers(buffer_device=device)
     model_new = model_new.to(device)
 
     loadable_state = {}
diff --git a/areal/tests/experimental/archon/torchrun/run_qwen3_parallelize.py b/areal/tests/experimental/archon/torchrun/run_qwen3_parallelize.py
index f3f90702c4..8fb2cedf87 100644
--- a/areal/tests/experimental/archon/torchrun/run_qwen3_parallelize.py
+++ b/areal/tests/experimental/archon/torchrun/run_qwen3_parallelize.py
@@ -103,7 +103,8 @@ def create_and_parallelize_model(
     """Create model, initialize weights, and apply parallelization."""
     torch.manual_seed(seed)
     model = Qwen3Model(model_args)
-    model.init_weights(device)
+    model.init_weights()
+    model.init_buffers(buffer_device=device)
     model = model.to(device)
 
     parallelize_qwen3(