diff --git a/jws/jws.go b/jws/jws.go
index c7811bc36..6908dd6bd 100644
--- a/jws/jws.go
+++ b/jws/jws.go
@@ -476,16 +476,6 @@ func (ctx *verifyCtx) verifyCompact(signed []byte) ([]byte, error) {
 		return nil, errors.Wrap(err, `failed extract from compact serialization format`)
 	}
 
-	verifyBuf := pool.GetBytesBuffer()
-	defer pool.ReleaseBytesBuffer(verifyBuf)
-
-	verifyBuf.Write(protected)
-	verifyBuf.WriteByte('.')
-	if len(payload) == 0 && ctx.detachedPayload != nil {
-		payload = ctx.detachedPayload
-	}
-	verifyBuf.Write(payload)
-
 	decodedSignature, err := base64.Decode(signature)
 	if err != nil {
 		return nil, errors.Wrap(err, `failed to decode signature`)
@@ -501,6 +491,20 @@ func (ctx *verifyCtx) verifyCompact(signed []byte) ([]byte, error) {
 		return nil, errors.Wrap(err, `failed to decode headers`)
 	}
 
+	verifyBuf := pool.GetBytesBuffer()
+	defer pool.ReleaseBytesBuffer(verifyBuf)
+
+	verifyBuf.Write(protected)
+	verifyBuf.WriteByte('.')
+	if len(payload) == 0 && ctx.detachedPayload != nil {
+		if getB64Value(hdr) {
+			payload = base64.Encode(ctx.detachedPayload)
+		} else {
+			payload = ctx.detachedPayload
+		}
+	}
+	verifyBuf.Write(payload)
+
 	if !ctx.useJKU {
 		if hdr.KeyID() != "" {
 			if jwkKey, ok := ctx.key.(jwk.Key); ok {
diff --git a/jws/jws_test.go b/jws/jws_test.go
index 626325a8b..23eee7dca 100644
--- a/jws/jws_test.go
+++ b/jws/jws_test.go
@@ -1778,3 +1778,20 @@ func TestJKU(t *testing.T) {
 		}
 	})
 }
+
+func TestGH681(t *testing.T) {
+	privkey, err := jwxtest.GenerateRsaKey()
+	if !assert.NoError(t, err, "failed to create private key") {
+		return
+	}
+
+	buf, err := jws.Sign(nil, jwa.RS256, privkey, jws.WithDetachedPayload([]byte("Lorem ipsum")))
+	if !assert.NoError(t, err, "failed to sign payload") {
+		return
+	}
+
+	_, err = jws.Verify(buf, jwa.RS256, &privkey.PublicKey, jws.WithDetachedPayload([]byte("Lorem ipsum")))
+	if !assert.NoError(t, err, "failed to verify JWS message") {
+		return
+	}
+}