diff --git a/NEWS.md b/NEWS.md
index 497a94fe39d13..c3f3f47d12b76 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -76,8 +76,6 @@ Language changes
   * Experimental feature: `x^n` for integer literals `n` (e.g. `x^3`
     or `x^-3`) is now lowered to `x^Val{n}`, to enable compile-time
     specialization for literal integer exponents ([#20530]).
-    `x^p` for `x::Number` and a literal `p=0,1,2,3` is now lowered to
-    `one(x)`, `x`, `x*x`, and `x*x*x`, respectively ([#20648]).
 
 Breaking changes
 ----------------
diff --git a/base/intfuncs.jl b/base/intfuncs.jl
index 69c0df7d11c17..bab711aa566c5 100644
--- a/base/intfuncs.jl
+++ b/base/intfuncs.jl
@@ -206,12 +206,18 @@ end
 @inline ^(x, p) = internal_pow(x, p)
 @inline internal_pow{p}(x, ::Type{Val{p}}) = x^p
 
+# Restrict inlining to hardware-supported arithmetic types, which
+# are fast enough to benefit from inlining.  This also makes it
+# easier to override ^ without having to override the Val method.
+const HWReal = Union{Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,Float32,Float64}
+const HWNumber = Union{HWReal, Complex{<:HWReal}, Rational{<:HWReal}}
+
 # inference.jl has complicated logic to inline x^2 and x^3 for
 # numeric types.  In terms of Val we can do it much more simply:
-@inline internal_pow(x::Number, ::Type{Val{0}}) = one(x)
-@inline internal_pow(x::Number, ::Type{Val{1}}) = x
-@inline internal_pow(x::Number, ::Type{Val{2}}) = x*x
-@inline internal_pow(x::Number, ::Type{Val{3}}) = x*x*x
+@inline internal_pow(x::HWNumber, ::Type{Val{0}}) = one(x)
+@inline internal_pow(x::HWNumber, ::Type{Val{1}}) = x
+@inline internal_pow(x::HWNumber, ::Type{Val{2}}) = x*x
+@inline internal_pow(x::HWNumber, ::Type{Val{3}}) = x*x*x
 
 # b^p mod m